2024-12-09 08:13:58,487 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-09 08:13:58,501 main DEBUG Took 0.011433 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 08:13:58,501 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 08:13:58,501 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 08:13:58,502 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 08:13:58,504 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,510 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 08:13:58,521 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,523 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,523 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,524 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,524 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,524 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,525 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,525 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,525 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,526 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,526 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,527 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,527 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,527 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,527 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,528 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,528 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,528 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,528 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,529 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,529 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,529 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,530 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:13:58,530 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,530 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 08:13:58,532 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:13:58,533 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 08:13:58,534 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 08:13:58,535 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 08:13:58,536 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 08:13:58,536 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 08:13:58,544 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 08:13:58,546 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 08:13:58,549 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 08:13:58,549 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 08:13:58,550 main DEBUG createAppenders(={Console}) 2024-12-09 08:13:58,551 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-09 08:13:58,551 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-09 08:13:58,551 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-09 08:13:58,552 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 08:13:58,552 main DEBUG OutputStream closed 2024-12-09 08:13:58,553 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 08:13:58,553 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 08:13:58,553 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-09 08:13:58,633 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 08:13:58,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 08:13:58,635 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 08:13:58,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 08:13:58,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 08:13:58,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 08:13:58,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 08:13:58,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 08:13:58,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 08:13:58,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 08:13:58,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 08:13:58,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 08:13:58,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 08:13:58,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 08:13:58,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 08:13:58,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 08:13:58,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 08:13:58,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 08:13:58,643 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 08:13:58,643 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-09 08:13:58,643 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 08:13:58,644 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-09T08:13:58,840 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4 2024-12-09 08:13:58,843 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 08:13:58,843 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T08:13:58,851 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-09T08:13:58,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T08:13:58,882 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/cluster_81f161a3-853c-b828-244c-d29c3e10103d, deleteOnExit=true 2024-12-09T08:13:58,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T08:13:58,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/test.cache.data in system properties and HBase conf 2024-12-09T08:13:58,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T08:13:58,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/hadoop.log.dir in system properties and HBase conf 2024-12-09T08:13:58,885 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T08:13:58,885 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T08:13:58,885 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T08:13:58,969 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T08:13:59,051 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T08:13:59,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T08:13:59,056 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T08:13:59,057 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T08:13:59,057 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T08:13:59,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T08:13:59,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T08:13:59,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T08:13:59,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T08:13:59,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T08:13:59,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/nfs.dump.dir in system properties and HBase conf 2024-12-09T08:13:59,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/java.io.tmpdir in system properties and HBase conf 2024-12-09T08:13:59,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T08:13:59,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T08:13:59,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T08:14:00,164 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T08:14:00,232 INFO [Time-limited test {}] log.Log(170): Logging initialized @2337ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T08:14:00,315 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:14:00,391 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T08:14:00,409 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T08:14:00,409 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T08:14:00,410 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T08:14:00,422 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:14:00,424 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/hadoop.log.dir/,AVAILABLE} 2024-12-09T08:14:00,425 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T08:14:00,585 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/java.io.tmpdir/jetty-localhost-33963-hadoop-hdfs-3_4_1-tests_jar-_-any-9178316184030502805/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T08:14:00,591 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33963} 2024-12-09T08:14:00,591 INFO [Time-limited test {}] server.Server(415): Started @2696ms 2024-12-09T08:14:01,230 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:14:01,239 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T08:14:01,241 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T08:14:01,242 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T08:14:01,242 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T08:14:01,246 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/hadoop.log.dir/,AVAILABLE} 2024-12-09T08:14:01,247 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T08:14:01,359 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/java.io.tmpdir/jetty-localhost-39627-hadoop-hdfs-3_4_1-tests_jar-_-any-7951133703143159590/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:14:01,360 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39627} 2024-12-09T08:14:01,360 INFO [Time-limited test {}] server.Server(415): Started @3466ms 2024-12-09T08:14:01,407 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T08:14:02,374 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/cluster_81f161a3-853c-b828-244c-d29c3e10103d/data/data1/current/BP-537288734-172.17.0.2-1733732039543/current, will proceed with Du for space computation calculation, 2024-12-09T08:14:02,374 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/cluster_81f161a3-853c-b828-244c-d29c3e10103d/data/data2/current/BP-537288734-172.17.0.2-1733732039543/current, will proceed with Du for space computation calculation, 2024-12-09T08:14:02,399 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T08:14:02,441 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc0633a3fbac92f6a with lease ID 0xf1a5d5f923b41c4e: Processing first storage report for DS-fb779b34-9a30-429a-8315-9225882ec69b from datanode DatanodeRegistration(127.0.0.1:32793, datanodeUuid=a5df0cd5-5ad6-4dc0-92ff-008714c4eaeb, infoPort=35825, infoSecurePort=0, ipcPort=42893, storageInfo=lv=-57;cid=testClusterID;nsid=832197630;c=1733732039543) 2024-12-09T08:14:02,442 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc0633a3fbac92f6a with lease ID 0xf1a5d5f923b41c4e: from storage DS-fb779b34-9a30-429a-8315-9225882ec69b node DatanodeRegistration(127.0.0.1:32793, datanodeUuid=a5df0cd5-5ad6-4dc0-92ff-008714c4eaeb, infoPort=35825, infoSecurePort=0, ipcPort=42893, storageInfo=lv=-57;cid=testClusterID;nsid=832197630;c=1733732039543), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T08:14:02,443 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc0633a3fbac92f6a with lease ID 0xf1a5d5f923b41c4e: Processing first storage report for DS-d81fde85-b1c4-4309-9b85-00bc5d5d7663 from datanode DatanodeRegistration(127.0.0.1:32793, datanodeUuid=a5df0cd5-5ad6-4dc0-92ff-008714c4eaeb, infoPort=35825, infoSecurePort=0, ipcPort=42893, storageInfo=lv=-57;cid=testClusterID;nsid=832197630;c=1733732039543) 2024-12-09T08:14:02,443 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc0633a3fbac92f6a with lease ID 0xf1a5d5f923b41c4e: from storage DS-d81fde85-b1c4-4309-9b85-00bc5d5d7663 node DatanodeRegistration(127.0.0.1:32793, datanodeUuid=a5df0cd5-5ad6-4dc0-92ff-008714c4eaeb, infoPort=35825, infoSecurePort=0, ipcPort=42893, storageInfo=lv=-57;cid=testClusterID;nsid=832197630;c=1733732039543), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T08:14:02,513 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4 2024-12-09T08:14:02,573 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/cluster_81f161a3-853c-b828-244c-d29c3e10103d/zookeeper_0, clientPort=55863, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/cluster_81f161a3-853c-b828-244c-d29c3e10103d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/cluster_81f161a3-853c-b828-244c-d29c3e10103d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T08:14:02,583 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55863 2024-12-09T08:14:02,592 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:14:02,594 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:14:02,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741825_1001 (size=7) 2024-12-09T08:14:03,210 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447 with version=8 2024-12-09T08:14:03,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/hbase-staging 2024-12-09T08:14:03,286 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T08:14:03,512 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5bda946059d1:0 server-side Connection retries=45 2024-12-09T08:14:03,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:14:03,521 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T08:14:03,525 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T08:14:03,525 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:14:03,525 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T08:14:03,639 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T08:14:03,689 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T08:14:03,696 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T08:14:03,699 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T08:14:03,721 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4924 (auto-detected) 2024-12-09T08:14:03,722 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T08:14:03,739 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45611 2024-12-09T08:14:03,758 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45611 connecting to ZooKeeper ensemble=127.0.0.1:55863 2024-12-09T08:14:03,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:456110x0, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T08:14:03,864 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45611-0x10009a38e040000 connected 2024-12-09T08:14:03,953 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:14:03,955 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:14:03,964 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:14:03,968 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447, hbase.cluster.distributed=false 2024-12-09T08:14:03,989 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T08:14:03,993 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45611 2024-12-09T08:14:03,994 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45611 2024-12-09T08:14:03,994 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45611 2024-12-09T08:14:03,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45611 2024-12-09T08:14:03,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45611 2024-12-09T08:14:04,081 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5bda946059d1:0 server-side Connection retries=45 2024-12-09T08:14:04,083 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:14:04,083 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T08:14:04,083 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T08:14:04,083 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:14:04,083 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T08:14:04,086 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T08:14:04,088 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T08:14:04,088 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43255 2024-12-09T08:14:04,090 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43255 connecting to ZooKeeper ensemble=127.0.0.1:55863 2024-12-09T08:14:04,091 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:14:04,094 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:14:04,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432550x0, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T08:14:04,120 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:432550x0, quorum=127.0.0.1:55863, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:14:04,120 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43255-0x10009a38e040001 connected 2024-12-09T08:14:04,124 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T08:14:04,131 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T08:14:04,134 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T08:14:04,139 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T08:14:04,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43255 2024-12-09T08:14:04,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43255 2024-12-09T08:14:04,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43255 2024-12-09T08:14:04,142 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43255 2024-12-09T08:14:04,142 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43255 2024-12-09T08:14:04,154 DEBUG [M:0;5bda946059d1:45611 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5bda946059d1:45611 2024-12-09T08:14:04,155 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/5bda946059d1,45611,1733732043366 2024-12-09T08:14:04,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:14:04,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:14:04,175 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/5bda946059d1,45611,1733732043366 2024-12-09T08:14:04,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:14:04,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T08:14:04,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:14:04,205 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T08:14:04,206 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/5bda946059d1,45611,1733732043366 from backup master directory 2024-12-09T08:14:04,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:14:04,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/5bda946059d1,45611,1733732043366 2024-12-09T08:14:04,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:14:04,215 WARN [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T08:14:04,215 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5bda946059d1,45611,1733732043366 2024-12-09T08:14:04,217 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T08:14:04,219 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T08:14:04,273 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/hbase.id] with ID: 2f412b64-5200-4500-97f2-3b4b9d069ff7 2024-12-09T08:14:04,273 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/.tmp/hbase.id 2024-12-09T08:14:04,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741826_1002 (size=42) 2024-12-09T08:14:04,687 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/.tmp/hbase.id]:[hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/hbase.id] 2024-12-09T08:14:04,739 INFO [master/5bda946059d1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:14:04,743 INFO [master/5bda946059d1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T08:14:04,760 INFO [master/5bda946059d1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-09T08:14:04,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:14:04,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:14:04,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741827_1003 (size=196) 2024-12-09T08:14:05,318 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:14:05,320 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T08:14:05,334 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:05,338 INFO [master/5bda946059d1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:14:05,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741828_1004 (size=1189) 2024-12-09T08:14:05,788 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store 2024-12-09T08:14:05,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741829_1005 (size=34) 2024-12-09T08:14:06,215 INFO [master/5bda946059d1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T08:14:06,219 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:14:06,219 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T08:14:06,220 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:14:06,220 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:14:06,221 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T08:14:06,221 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:14:06,221 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:14:06,222 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733732046219Disabling compacts and flushes for region at 1733732046219Disabling writes for close at 1733732046221 (+2 ms)Writing region close event to WAL at 1733732046221Closed at 1733732046221 2024-12-09T08:14:06,224 WARN [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/.initializing 2024-12-09T08:14:06,224 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/WALs/5bda946059d1,45611,1733732043366 2024-12-09T08:14:06,232 INFO [master/5bda946059d1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:14:06,244 INFO [master/5bda946059d1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C45611%2C1733732043366, suffix=, logDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/WALs/5bda946059d1,45611,1733732043366, archiveDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/oldWALs, maxLogs=10 2024-12-09T08:14:06,263 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/WALs/5bda946059d1,45611,1733732043366/5bda946059d1%2C45611%2C1733732043366.1733732046248, exclude list is [], retry=0 2024-12-09T08:14:06,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32793,DS-fb779b34-9a30-429a-8315-9225882ec69b,DISK] 2024-12-09T08:14:06,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T08:14:06,308 INFO [master/5bda946059d1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/WALs/5bda946059d1,45611,1733732043366/5bda946059d1%2C45611%2C1733732043366.1733732046248 2024-12-09T08:14:06,309 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35825:35825)] 2024-12-09T08:14:06,310 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:14:06,310 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:14:06,313 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:14:06,314 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:14:06,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:14:06,365 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T08:14:06,368 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:06,370 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:14:06,371 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:14:06,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T08:14:06,374 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:06,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:14:06,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:14:06,378 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T08:14:06,378 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:06,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:14:06,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:14:06,381 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T08:14:06,382 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:06,382 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:14:06,383 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:14:06,387 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:14:06,388 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:14:06,392 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:14:06,393 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:14:06,398 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T08:14:06,402 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:14:06,407 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:14:06,408 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60795992, jitterRate=-0.09406912326812744}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T08:14:06,413 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733732046324Initializing all the Stores at 1733732046326 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732046327 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732046327Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732046327Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732046328 (+1 ms)Cleaning up temporary data from old regions at 1733732046394 (+66 ms)Region opened successfully at 1733732046413 (+19 ms) 2024-12-09T08:14:06,414 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T08:14:06,444 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f84067, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5bda946059d1/172.17.0.2:0 2024-12-09T08:14:06,470 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T08:14:06,479 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T08:14:06,480 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T08:14:06,482 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T08:14:06,483 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T08:14:06,487 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T08:14:06,488 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T08:14:06,509 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T08:14:06,517 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T08:14:06,572 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T08:14:06,575 INFO [master/5bda946059d1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T08:14:06,577 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T08:14:06,582 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T08:14:06,585 INFO [master/5bda946059d1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T08:14:06,589 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T08:14:06,601 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T08:14:06,603 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T08:14:06,614 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T08:14:06,636 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T08:14:06,645 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T08:14:06,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T08:14:06,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T08:14:06,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:14:06,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:14:06,661 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5bda946059d1,45611,1733732043366, sessionid=0x10009a38e040000, setting cluster-up flag (Was=false) 2024-12-09T08:14:06,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:14:06,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:14:06,720 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T08:14:06,725 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5bda946059d1,45611,1733732043366 2024-12-09T08:14:06,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:14:06,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:14:06,782 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T08:14:06,786 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5bda946059d1,45611,1733732043366 2024-12-09T08:14:06,794 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T08:14:06,849 INFO [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(746): ClusterId : 2f412b64-5200-4500-97f2-3b4b9d069ff7 2024-12-09T08:14:06,852 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T08:14:06,855 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T08:14:06,864 INFO [master/5bda946059d1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T08:14:06,866 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T08:14:06,866 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T08:14:06,870 INFO [master/5bda946059d1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T08:14:06,875 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5bda946059d1,45611,1733732043366 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T08:14:06,879 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T08:14:06,879 DEBUG [RS:0;5bda946059d1:43255 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a4acb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5bda946059d1/172.17.0.2:0 2024-12-09T08:14:06,883 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5bda946059d1:0, corePoolSize=5, maxPoolSize=5 2024-12-09T08:14:06,883 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5bda946059d1:0, corePoolSize=5, maxPoolSize=5 2024-12-09T08:14:06,884 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5bda946059d1:0, corePoolSize=5, maxPoolSize=5 2024-12-09T08:14:06,884 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5bda946059d1:0, corePoolSize=5, maxPoolSize=5 2024-12-09T08:14:06,884 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5bda946059d1:0, corePoolSize=10, maxPoolSize=10 2024-12-09T08:14:06,884 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:06,884 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5bda946059d1:0, corePoolSize=2, maxPoolSize=2 2024-12-09T08:14:06,885 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:06,889 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733732076889 2024-12-09T08:14:06,890 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T08:14:06,890 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T08:14:06,891 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T08:14:06,893 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T08:14:06,894 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5bda946059d1:43255 2024-12-09T08:14:06,897 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T08:14:06,897 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T08:14:06,897 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:06,897 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T08:14:06,897 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T08:14:06,897 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T08:14:06,898 INFO [RS:0;5bda946059d1:43255 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T08:14:06,899 INFO [RS:0;5bda946059d1:43255 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T08:14:06,899 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T08:14:06,898 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:06,902 INFO [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(2659): reportForDuty to master=5bda946059d1,45611,1733732043366 with port=43255, startcode=1733732044053 2024-12-09T08:14:06,904 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T08:14:06,905 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T08:14:06,906 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T08:14:06,907 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T08:14:06,908 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T08:14:06,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741831_1007 (size=1321) 2024-12-09T08:14:06,909 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.large.0-1733732046909,5,FailOnTimeoutGroup] 2024-12-09T08:14:06,910 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.small.0-1733732046910,5,FailOnTimeoutGroup] 2024-12-09T08:14:06,910 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:06,910 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T08:14:06,912 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:06,912 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:06,913 DEBUG [RS:0;5bda946059d1:43255 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T08:14:06,945 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48283, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T08:14:06,951 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45611 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5bda946059d1,43255,1733732044053 2024-12-09T08:14:06,953 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45611 {}] master.ServerManager(517): Registering regionserver=5bda946059d1,43255,1733732044053 2024-12-09T08:14:06,967 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447 2024-12-09T08:14:06,967 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38367 2024-12-09T08:14:06,967 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T08:14:06,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T08:14:06,983 DEBUG [RS:0;5bda946059d1:43255 {}] zookeeper.ZKUtil(111): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5bda946059d1,43255,1733732044053 2024-12-09T08:14:06,983 WARN [RS:0;5bda946059d1:43255 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T08:14:06,983 INFO [RS:0;5bda946059d1:43255 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:14:06,984 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/WALs/5bda946059d1,43255,1733732044053 2024-12-09T08:14:06,985 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5bda946059d1,43255,1733732044053] 2024-12-09T08:14:07,006 INFO [RS:0;5bda946059d1:43255 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T08:14:07,018 INFO [RS:0;5bda946059d1:43255 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T08:14:07,022 INFO [RS:0;5bda946059d1:43255 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T08:14:07,022 INFO [RS:0;5bda946059d1:43255 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:07,023 INFO [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T08:14:07,027 INFO [RS:0;5bda946059d1:43255 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T08:14:07,028 INFO [RS:0;5bda946059d1:43255 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:07,029 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:07,029 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:07,029 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:07,029 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:07,029 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:07,029 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0, corePoolSize=2, maxPoolSize=2 2024-12-09T08:14:07,029 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:07,030 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:07,030 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:07,030 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:07,030 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:07,030 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:14:07,030 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:14:07,030 DEBUG [RS:0;5bda946059d1:43255 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:14:07,031 INFO [RS:0;5bda946059d1:43255 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:07,032 INFO [RS:0;5bda946059d1:43255 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:07,032 INFO [RS:0;5bda946059d1:43255 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:07,032 INFO [RS:0;5bda946059d1:43255 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:07,032 INFO [RS:0;5bda946059d1:43255 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:07,032 INFO [RS:0;5bda946059d1:43255 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,43255,1733732044053-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T08:14:07,049 INFO [RS:0;5bda946059d1:43255 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T08:14:07,051 INFO [RS:0;5bda946059d1:43255 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,43255,1733732044053-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:07,051 INFO [RS:0;5bda946059d1:43255 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:07,051 INFO [RS:0;5bda946059d1:43255 {}] regionserver.Replication(171): 5bda946059d1,43255,1733732044053 started 2024-12-09T08:14:07,068 INFO [RS:0;5bda946059d1:43255 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:07,068 INFO [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1482): Serving as 5bda946059d1,43255,1733732044053, RpcServer on 5bda946059d1/172.17.0.2:43255, sessionid=0x10009a38e040001 2024-12-09T08:14:07,069 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T08:14:07,069 DEBUG [RS:0;5bda946059d1:43255 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5bda946059d1,43255,1733732044053 2024-12-09T08:14:07,069 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,43255,1733732044053' 2024-12-09T08:14:07,070 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T08:14:07,070 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T08:14:07,071 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T08:14:07,071 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T08:14:07,071 DEBUG [RS:0;5bda946059d1:43255 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5bda946059d1,43255,1733732044053 2024-12-09T08:14:07,071 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,43255,1733732044053' 2024-12-09T08:14:07,072 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T08:14:07,072 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T08:14:07,073 DEBUG [RS:0;5bda946059d1:43255 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T08:14:07,073 INFO [RS:0;5bda946059d1:43255 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T08:14:07,073 INFO [RS:0;5bda946059d1:43255 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T08:14:07,182 INFO [RS:0;5bda946059d1:43255 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:14:07,186 INFO [RS:0;5bda946059d1:43255 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C43255%2C1733732044053, suffix=, logDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/WALs/5bda946059d1,43255,1733732044053, archiveDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/oldWALs, maxLogs=32 2024-12-09T08:14:07,199 DEBUG [RS:0;5bda946059d1:43255 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/WALs/5bda946059d1,43255,1733732044053/5bda946059d1%2C43255%2C1733732044053.1733732047188, exclude list is [], retry=0 2024-12-09T08:14:07,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32793,DS-fb779b34-9a30-429a-8315-9225882ec69b,DISK] 2024-12-09T08:14:07,207 INFO [RS:0;5bda946059d1:43255 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/WALs/5bda946059d1,43255,1733732044053/5bda946059d1%2C43255%2C1733732044053.1733732047188 2024-12-09T08:14:07,207 DEBUG [RS:0;5bda946059d1:43255 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35825:35825)] 2024-12-09T08:14:07,314 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T08:14:07,315 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447 2024-12-09T08:14:07,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741833_1009 (size=32) 2024-12-09T08:14:07,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:14:07,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T08:14:07,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T08:14:07,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:07,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:14:07,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T08:14:07,750 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T08:14:07,750 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:07,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:14:07,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T08:14:07,754 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T08:14:07,754 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:07,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:14:07,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T08:14:07,758 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T08:14:07,758 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:07,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:14:07,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T08:14:07,760 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740 2024-12-09T08:14:07,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740 2024-12-09T08:14:07,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T08:14:07,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T08:14:07,765 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:14:07,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T08:14:07,771 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:14:07,772 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72970198, jitterRate=0.08734068274497986}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:14:07,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733732047735Initializing all the Stores at 1733732047739 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732047740 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732047740Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732047740Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732047740Cleaning up temporary data from old regions at 1733732047764 (+24 ms)Region opened successfully at 1733732047775 (+11 ms) 2024-12-09T08:14:07,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T08:14:07,776 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T08:14:07,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T08:14:07,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T08:14:07,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T08:14:07,777 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T08:14:07,778 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733732047776Disabling compacts and flushes for region at 1733732047776Disabling writes for close at 1733732047776Writing region close event to WAL at 1733732047777 (+1 ms)Closed at 1733732047777 2024-12-09T08:14:07,781 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T08:14:07,781 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T08:14:07,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T08:14:07,795 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T08:14:07,797 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T08:14:07,954 DEBUG [5bda946059d1:45611 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T08:14:07,965 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:14:07,970 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5bda946059d1,43255,1733732044053, state=OPENING 2024-12-09T08:14:08,046 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T08:14:08,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:14:08,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:14:08,059 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:14:08,059 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:14:08,063 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T08:14:08,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:14:08,243 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T08:14:08,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33379, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T08:14:08,255 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T08:14:08,255 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:14:08,256 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T08:14:08,259 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C43255%2C1733732044053.meta, suffix=.meta, logDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/WALs/5bda946059d1,43255,1733732044053, archiveDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/oldWALs, maxLogs=32 2024-12-09T08:14:08,273 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/WALs/5bda946059d1,43255,1733732044053/5bda946059d1%2C43255%2C1733732044053.meta.1733732048260.meta, exclude list is [], retry=0 2024-12-09T08:14:08,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32793,DS-fb779b34-9a30-429a-8315-9225882ec69b,DISK] 2024-12-09T08:14:08,280 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/WALs/5bda946059d1,43255,1733732044053/5bda946059d1%2C43255%2C1733732044053.meta.1733732048260.meta 2024-12-09T08:14:08,281 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35825:35825)] 2024-12-09T08:14:08,281 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:14:08,283 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T08:14:08,285 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T08:14:08,289 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T08:14:08,293 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T08:14:08,293 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:14:08,293 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T08:14:08,294 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T08:14:08,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T08:14:08,298 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T08:14:08,298 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:08,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:14:08,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T08:14:08,301 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T08:14:08,301 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:08,302 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:14:08,302 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T08:14:08,304 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T08:14:08,304 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:08,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:14:08,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T08:14:08,307 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T08:14:08,307 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:08,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:14:08,308 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T08:14:08,310 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740 2024-12-09T08:14:08,312 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740 2024-12-09T08:14:08,315 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T08:14:08,315 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T08:14:08,316 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:14:08,319 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T08:14:08,321 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62674505, jitterRate=-0.0660770982503891}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:14:08,321 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T08:14:08,323 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733732048294Writing region info on filesystem at 1733732048294Initializing all the Stores at 1733732048296 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732048296Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732048296Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732048296Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732048296Cleaning up temporary data from old regions at 1733732048315 (+19 ms)Running coprocessor post-open hooks at 1733732048321 (+6 ms)Region opened successfully at 1733732048323 (+2 ms) 2024-12-09T08:14:08,330 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733732048234 2024-12-09T08:14:08,341 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T08:14:08,342 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T08:14:08,343 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:14:08,346 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5bda946059d1,43255,1733732044053, state=OPEN 2024-12-09T08:14:08,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T08:14:08,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T08:14:08,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:14:08,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:14:08,502 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:08,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T08:14:08,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5bda946059d1,43255,1733732044053 in 438 msec 2024-12-09T08:14:08,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T08:14:08,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 725 msec 2024-12-09T08:14:08,518 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T08:14:08,518 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T08:14:08,538 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:08,539 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:08,555 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:08,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58725, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:08,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7550 sec 2024-12-09T08:14:08,578 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733732048578, completionTime=-1 2024-12-09T08:14:08,596 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T08:14:08,597 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T08:14:08,626 INFO [master/5bda946059d1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-09T08:14:08,627 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733732108627 2024-12-09T08:14:08,627 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733732168627 2024-12-09T08:14:08,627 INFO [master/5bda946059d1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-12-09T08:14:08,629 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,45611,1733732043366-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:08,630 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,45611,1733732043366-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:08,630 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,45611,1733732043366-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:08,631 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5bda946059d1:45611, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:08,632 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:08,633 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:08,637 DEBUG [master/5bda946059d1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T08:14:08,656 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.441sec 2024-12-09T08:14:08,657 INFO [master/5bda946059d1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T08:14:08,658 INFO [master/5bda946059d1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T08:14:08,659 INFO [master/5bda946059d1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T08:14:08,659 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T08:14:08,659 INFO [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T08:14:08,660 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,45611,1733732043366-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T08:14:08,660 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,45611,1733732043366-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T08:14:08,668 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T08:14:08,669 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T08:14:08,669 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,45611,1733732043366-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:14:08,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:08,765 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T08:14:08,765 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T08:14:08,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:08,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:08,776 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:08,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:08,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d86f895, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:08,785 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:08,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:08,788 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:08,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:08,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:08,797 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:08,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5bda946059d1,45611,1733732043366 2024-12-09T08:14:08,829 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=264, ProcessCount=11, AvailableMemoryMB=4645 2024-12-09T08:14:08,836 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T08:14:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5bda946059d1,45611,1733732043366 2024-12-09T08:14:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@65d48f8e 2024-12-09T08:14:08,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T08:14:08,846 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T08:14:08,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T08:14:08,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:14:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T08:14:08,865 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T08:14:08,867 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-09T08:14:08,867 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:08,869 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T08:14:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:14:08,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741835_1011 (size=1061) 2024-12-09T08:14:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:14:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:14:09,286 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447 2024-12-09T08:14:09,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741836_1012 (size=53) 2024-12-09T08:14:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:14:09,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:14:09,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 589b933f5d38176578d90952a67fb0a1, disabling compactions & flushes 2024-12-09T08:14:09,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:09,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:09,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. after waiting 0 ms 2024-12-09T08:14:09,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:09,699 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:09,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 589b933f5d38176578d90952a67fb0a1: Waiting for close lock at 1733732049698Disabling compacts and flushes for region at 1733732049698Disabling writes for close at 1733732049699 (+1 ms)Writing region close event to WAL at 1733732049699Closed at 1733732049699 2024-12-09T08:14:09,704 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T08:14:09,710 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733732049705"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733732049705"}]},"ts":"1733732049705"} 2024-12-09T08:14:09,716 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T08:14:09,718 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T08:14:09,721 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732049718"}]},"ts":"1733732049718"} 2024-12-09T08:14:09,726 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T08:14:09,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=589b933f5d38176578d90952a67fb0a1, ASSIGN}] 2024-12-09T08:14:09,731 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=589b933f5d38176578d90952a67fb0a1, ASSIGN 2024-12-09T08:14:09,733 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=589b933f5d38176578d90952a67fb0a1, ASSIGN; state=OFFLINE, location=5bda946059d1,43255,1733732044053; forceNewPlan=false, retain=false 2024-12-09T08:14:09,884 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=589b933f5d38176578d90952a67fb0a1, regionState=OPENING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:14:09,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=589b933f5d38176578d90952a67fb0a1, ASSIGN because future has completed 2024-12-09T08:14:09,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:14:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:14:10,055 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:10,056 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:14:10,056 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,057 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:14:10,057 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,057 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,060 INFO [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,063 INFO [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:14:10,063 INFO [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 589b933f5d38176578d90952a67fb0a1 columnFamilyName A 2024-12-09T08:14:10,064 DEBUG [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:10,065 INFO [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] regionserver.HStore(327): Store=589b933f5d38176578d90952a67fb0a1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:14:10,065 INFO [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,067 INFO [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:14:10,067 INFO [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 589b933f5d38176578d90952a67fb0a1 columnFamilyName B 2024-12-09T08:14:10,068 DEBUG [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:10,069 INFO [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] regionserver.HStore(327): Store=589b933f5d38176578d90952a67fb0a1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:14:10,069 INFO [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,071 INFO [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:14:10,072 INFO [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 589b933f5d38176578d90952a67fb0a1 columnFamilyName C 2024-12-09T08:14:10,072 DEBUG [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:10,073 INFO [StoreOpener-589b933f5d38176578d90952a67fb0a1-1 {}] regionserver.HStore(327): Store=589b933f5d38176578d90952a67fb0a1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:14:10,073 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:10,073 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,075 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,075 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,077 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,077 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,078 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:14:10,080 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,083 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:14:10,084 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 589b933f5d38176578d90952a67fb0a1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59777060, jitterRate=-0.10925239324569702}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:14:10,084 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:10,085 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 589b933f5d38176578d90952a67fb0a1: Running coprocessor pre-open hook at 1733732050057Writing region info on filesystem at 1733732050057Initializing all the Stores at 1733732050059 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732050059Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732050060 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732050060Cleaning up temporary data from old regions at 1733732050077 (+17 ms)Running coprocessor post-open hooks at 1733732050084 (+7 ms)Region opened successfully at 1733732050085 (+1 ms) 2024-12-09T08:14:10,087 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., pid=6, masterSystemTime=1733732050046 2024-12-09T08:14:10,089 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:10,090 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:10,091 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=589b933f5d38176578d90952a67fb0a1, regionState=OPEN, openSeqNum=2, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:14:10,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:14:10,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T08:14:10,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 in 205 msec 2024-12-09T08:14:10,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T08:14:10,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=589b933f5d38176578d90952a67fb0a1, ASSIGN in 372 msec 2024-12-09T08:14:10,105 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T08:14:10,106 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732050105"}]},"ts":"1733732050105"} 2024-12-09T08:14:10,109 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T08:14:10,111 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T08:14:10,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2550 sec 2024-12-09T08:14:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:14:11,023 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:11,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:11,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:11,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5de9b018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:11,036 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc499f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,038 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:11,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:11,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:11,044 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T08:14:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 5bda946059d1,45611,1733732043366 2024-12-09T08:14:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@36f9a744 2024-12-09T08:14:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T08:14:11,050 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T08:14:11,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,054 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:11,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:11,055 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d22a118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:11,059 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:11,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:11,062 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:11,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:11,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:11,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ed1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:11,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:11,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:11,065 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,067 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:11,067 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:11,068 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef39c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:11,072 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f3cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,074 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:11,074 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:11,075 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:11,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:11,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ac3455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:11,079 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d699c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,081 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:11,081 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:11,082 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:11,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3abd488a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:11,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:11,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:11,084 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103656b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,086 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:11,086 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:11,088 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:11,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:11,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:11,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbf9ca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:11,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:11,091 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19664801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,092 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:11,093 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:11,093 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@614ff77d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:11,095 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:11,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c26f37b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:11,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:11,099 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:11,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa1e09e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:11,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:11,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:11,102 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788043de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,109 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:11,110 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:11,111 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:11,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:11,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d5cf1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:11,113 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@caecd72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:11,121 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:11,121 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:11,121 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:11,122 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:11,123 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:11,125 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:11,126 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:11,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:11,127 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:11,127 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:11,128 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:11,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:11,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:11,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:11,130 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:11,130 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:11,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:11,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:11,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:11,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:11,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:11,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:14:11,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:11,137 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:14:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:14:11,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:14:11,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:14:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-09T08:14:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:14:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:14:11,148 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:11,151 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:11,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:14:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T08:14:11,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:11,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:14:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:11,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:14:11,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:11,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:11,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:11,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:11,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:11,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T08:14:11,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/56bd77eeb6094fc0a71f0db8f82791dd is 50, key is test_row_0/A:col10/1733732051178/Put/seqid=0 2024-12-09T08:14:11,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T08:14:11,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741837_1013 (size=12001) 2024-12-09T08:14:11,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/56bd77eeb6094fc0a71f0db8f82791dd 2024-12-09T08:14:11,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:11,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:11,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:11,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:11,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:11,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732111336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732111345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732111353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732051290, max timestamp=1733732051358Num uniques 859; 2024-12-09T08:14:11,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733732051290, max timestamp=1733732051352Num uniques 859; 2024-12-09T08:14:11,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T08:14:11,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:14:11,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732051290, max timestamp=1733732051358Num uniques 859; 2024-12-09T08:14:11,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:14:11,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/b170fc7aa1d64df8b95911a79efea616 is 50, key is test_row_0/B:col10/1733732051178/Put/seqid=0 2024-12-09T08:14:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T08:14:11,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741838_1014 (size=12001) 2024-12-09T08:14:11,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/b170fc7aa1d64df8b95911a79efea616 2024-12-09T08:14:11,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732111510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732111514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732111522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732111522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732111524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/67d3766a9fad43d887dbcd62ecd9acd3 is 50, key is test_row_0/C:col10/1733732051178/Put/seqid=0 2024-12-09T08:14:11,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T08:14:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:11,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:11,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741839_1015 (size=12001) 2024-12-09T08:14:11,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/67d3766a9fad43d887dbcd62ecd9acd3 2024-12-09T08:14:11,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/56bd77eeb6094fc0a71f0db8f82791dd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/56bd77eeb6094fc0a71f0db8f82791dd 2024-12-09T08:14:11,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/56bd77eeb6094fc0a71f0db8f82791dd, entries=150, sequenceid=12, filesize=11.7 K 2024-12-09T08:14:11,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732111622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/b170fc7aa1d64df8b95911a79efea616 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/b170fc7aa1d64df8b95911a79efea616 2024-12-09T08:14:11,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/b170fc7aa1d64df8b95911a79efea616, entries=150, sequenceid=12, filesize=11.7 K 2024-12-09T08:14:11,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/67d3766a9fad43d887dbcd62ecd9acd3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/67d3766a9fad43d887dbcd62ecd9acd3 2024-12-09T08:14:11,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732111632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/67d3766a9fad43d887dbcd62ecd9acd3, entries=150, sequenceid=12, filesize=11.7 K 2024-12-09T08:14:11,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 589b933f5d38176578d90952a67fb0a1 in 480ms, sequenceid=12, compaction requested=false 2024-12-09T08:14:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T08:14:11,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:11,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T08:14:11,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:11,717 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-12-09T08:14:11,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:11,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:11,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:11,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:11,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:11,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:11,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:11,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/c0d0ba0fdff74453b4d6e39d773dac17 is 50, key is test_row_0/A:col10/1733732051499/Put/seqid=0 2024-12-09T08:14:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T08:14:11,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741840_1016 (size=12001) 2024-12-09T08:14:11,798 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/c0d0ba0fdff74453b4d6e39d773dac17 2024-12-09T08:14:11,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732111796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/88343ec5de3544fa820fa28cebcf30cb is 50, key is test_row_0/B:col10/1733732051499/Put/seqid=0 2024-12-09T08:14:11,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732111805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732111812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741841_1017 (size=12001) 2024-12-09T08:14:11,850 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/88343ec5de3544fa820fa28cebcf30cb 2024-12-09T08:14:11,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732111843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732111882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/105d5dd080844e3e91ef5084b0b099c8 is 50, key is test_row_0/C:col10/1733732051499/Put/seqid=0 2024-12-09T08:14:11,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741842_1018 (size=12001) 2024-12-09T08:14:11,938 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/105d5dd080844e3e91ef5084b0b099c8 2024-12-09T08:14:11,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732111933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732111934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732111934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:11,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/c0d0ba0fdff74453b4d6e39d773dac17 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/c0d0ba0fdff74453b4d6e39d773dac17 2024-12-09T08:14:11,971 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/c0d0ba0fdff74453b4d6e39d773dac17, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T08:14:11,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/88343ec5de3544fa820fa28cebcf30cb as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/88343ec5de3544fa820fa28cebcf30cb 2024-12-09T08:14:11,989 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/88343ec5de3544fa820fa28cebcf30cb, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T08:14:11,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/105d5dd080844e3e91ef5084b0b099c8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/105d5dd080844e3e91ef5084b0b099c8 2024-12-09T08:14:12,009 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/105d5dd080844e3e91ef5084b0b099c8, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T08:14:12,011 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=53.67 KB/54960 for 589b933f5d38176578d90952a67fb0a1 in 294ms, sequenceid=48, compaction requested=false 2024-12-09T08:14:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-09T08:14:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-09T08:14:12,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T08:14:12,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 862 msec 2024-12-09T08:14:12,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 886 msec 2024-12-09T08:14:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:12,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T08:14:12,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:12,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:12,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:12,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:12,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:12,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:12,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/7f865960dbcb4569bb0a6275f5f0806f is 50, key is test_row_0/A:col10/1733732052157/Put/seqid=0 2024-12-09T08:14:12,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741843_1019 (size=12001) 2024-12-09T08:14:12,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/7f865960dbcb4569bb0a6275f5f0806f 2024-12-09T08:14:12,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732112235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732112239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732112246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732112248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732112249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/f9c889ce7de748b28917afc6445ad0bd is 50, key is test_row_0/B:col10/1733732052157/Put/seqid=0 2024-12-09T08:14:12,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741844_1020 (size=12001) 2024-12-09T08:14:12,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/f9c889ce7de748b28917afc6445ad0bd 2024-12-09T08:14:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T08:14:12,292 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:12,293 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-09T08:14:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T08:14:12,297 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:12,300 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:12,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:12,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/dfe131f9ff5f492ca1c1ee5176d85947 is 50, key is test_row_0/C:col10/1733732052157/Put/seqid=0 2024-12-09T08:14:12,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741845_1021 (size=12001) 2024-12-09T08:14:12,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732112352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732112362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732112363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732112363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732112365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T08:14:12,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T08:14:12,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:12,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:12,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:12,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:12,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:12,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732112562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732112572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732112572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732112572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732112582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:12,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T08:14:12,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T08:14:12,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:12,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:12,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:12,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:12,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:12,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/dfe131f9ff5f492ca1c1ee5176d85947 2024-12-09T08:14:12,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/7f865960dbcb4569bb0a6275f5f0806f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7f865960dbcb4569bb0a6275f5f0806f 2024-12-09T08:14:12,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T08:14:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:12,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:12,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:12,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7f865960dbcb4569bb0a6275f5f0806f, entries=150, sequenceid=61, filesize=11.7 K 2024-12-09T08:14:12,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/f9c889ce7de748b28917afc6445ad0bd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f9c889ce7de748b28917afc6445ad0bd 2024-12-09T08:14:12,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f9c889ce7de748b28917afc6445ad0bd, entries=150, sequenceid=61, filesize=11.7 K 2024-12-09T08:14:12,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/dfe131f9ff5f492ca1c1ee5176d85947 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/dfe131f9ff5f492ca1c1ee5176d85947 2024-12-09T08:14:12,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/dfe131f9ff5f492ca1c1ee5176d85947, entries=150, sequenceid=61, filesize=11.7 K 2024-12-09T08:14:12,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 589b933f5d38176578d90952a67fb0a1 in 653ms, sequenceid=61, compaction requested=true 2024-12-09T08:14:12,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:12,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:12,844 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:12,846 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:12,849 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:12,851 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:12,852 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:12,853 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/b170fc7aa1d64df8b95911a79efea616, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/88343ec5de3544fa820fa28cebcf30cb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f9c889ce7de748b28917afc6445ad0bd] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=35.2 K 2024-12-09T08:14:12,855 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting b170fc7aa1d64df8b95911a79efea616, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733732051152 2024-12-09T08:14:12,856 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:12,856 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 88343ec5de3544fa820fa28cebcf30cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733732051447 2024-12-09T08:14:12,857 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/A is initiating minor compaction (all files) 2024-12-09T08:14:12,857 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/A in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:12,857 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/56bd77eeb6094fc0a71f0db8f82791dd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/c0d0ba0fdff74453b4d6e39d773dac17, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7f865960dbcb4569bb0a6275f5f0806f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=35.2 K 2024-12-09T08:14:12,857 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting f9c889ce7de748b28917afc6445ad0bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733732051779 2024-12-09T08:14:12,860 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56bd77eeb6094fc0a71f0db8f82791dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733732051152 2024-12-09T08:14:12,863 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0d0ba0fdff74453b4d6e39d773dac17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733732051447 2024-12-09T08:14:12,865 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f865960dbcb4569bb0a6275f5f0806f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733732051779 2024-12-09T08:14:12,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732052242, max timestamp=1733732052880Num uniques 859; 2024-12-09T08:14:12,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:14:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:12,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732052242, max timestamp=1733732052880Num uniques 859; 2024-12-09T08:14:12,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:14:12,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-09T08:14:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:12,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/bd02bc1c46ee42c3ad71f25c1e0b9ac8 is 50, key is test_row_0/A:col10/1733732052899/Put/seqid=0 2024-12-09T08:14:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T08:14:12,931 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:12,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T08:14:12,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:12,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:12,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:12,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:12,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:12,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:12,952 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/1cceb52d8ec24e3aad586029082dd846 is 50, key is test_row_0/B:col10/1733732052157/Put/seqid=0 2024-12-09T08:14:12,983 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#A#compaction#9 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:12,984 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/7d7673ce2efe4e90a58b562d9215a65a is 50, key is test_row_0/A:col10/1733732052157/Put/seqid=0 2024-12-09T08:14:12,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741846_1022 (size=14341) 2024-12-09T08:14:12,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/bd02bc1c46ee42c3ad71f25c1e0b9ac8 2024-12-09T08:14:13,003 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T08:14:13,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741847_1023 (size=12104) 2024-12-09T08:14:13,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732112995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732113001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732113002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732113009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732113018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,033 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/1cceb52d8ec24e3aad586029082dd846 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1cceb52d8ec24e3aad586029082dd846 2024-12-09T08:14:13,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/93e72dc314c24252a633c02e1f5fcdb1 is 50, key is test_row_0/B:col10/1733732052899/Put/seqid=0 2024-12-09T08:14:13,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741848_1024 (size=12104) 2024-12-09T08:14:13,063 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into 1cceb52d8ec24e3aad586029082dd846(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:13,063 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:13,066 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=13, startTime=1733732052842; duration=0sec 2024-12-09T08:14:13,066 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:13,067 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:13,067 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:13,075 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:13,075 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:13,075 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,075 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/67d3766a9fad43d887dbcd62ecd9acd3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/105d5dd080844e3e91ef5084b0b099c8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/dfe131f9ff5f492ca1c1ee5176d85947] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=35.2 K 2024-12-09T08:14:13,077 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d3766a9fad43d887dbcd62ecd9acd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733732051152 2024-12-09T08:14:13,078 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 105d5dd080844e3e91ef5084b0b099c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733732051447 2024-12-09T08:14:13,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741849_1025 (size=12001) 2024-12-09T08:14:13,079 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe131f9ff5f492ca1c1ee5176d85947, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733732051779 2024-12-09T08:14:13,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T08:14:13,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:13,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,100 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:13,101 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/6abc9fe9495d4ddaabaff11bc3e689d2 is 50, key is test_row_0/C:col10/1733732052157/Put/seqid=0 2024-12-09T08:14:13,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741850_1026 (size=12104) 2024-12-09T08:14:13,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732113121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732113132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732113132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732113132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732113133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T08:14:13,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:13,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732113332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732113352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732113353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732113353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732113353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T08:14:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T08:14:13,534 INFO [AsyncFSWAL-0-hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData-prefix:5bda946059d1,45611,1733732043366 {}] wal.AbstractFSWAL(1368): Slow sync cost: 119 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32793,DS-fb779b34-9a30-429a-8315-9225882ec69b,DISK]] 2024-12-09T08:14:13,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/93e72dc314c24252a633c02e1f5fcdb1 2024-12-09T08:14:13,558 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/7d7673ce2efe4e90a58b562d9215a65a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7d7673ce2efe4e90a58b562d9215a65a 2024-12-09T08:14:13,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/7cec363994554cbe95448910bca60b7c is 50, key is test_row_0/C:col10/1733732052899/Put/seqid=0 2024-12-09T08:14:13,571 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/6abc9fe9495d4ddaabaff11bc3e689d2 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/6abc9fe9495d4ddaabaff11bc3e689d2 2024-12-09T08:14:13,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741851_1027 (size=12001) 2024-12-09T08:14:13,580 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/A of 589b933f5d38176578d90952a67fb0a1 into 7d7673ce2efe4e90a58b562d9215a65a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:13,580 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:13,581 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/A, priority=13, startTime=1733732052820; duration=0sec 2024-12-09T08:14:13,581 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:13,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/7cec363994554cbe95448910bca60b7c 2024-12-09T08:14:13,581 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:13,589 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into 6abc9fe9495d4ddaabaff11bc3e689d2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:13,589 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:13,589 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=13, startTime=1733732052842; duration=0sec 2024-12-09T08:14:13,589 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:13,589 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:13,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/bd02bc1c46ee42c3ad71f25c1e0b9ac8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bd02bc1c46ee42c3ad71f25c1e0b9ac8 2024-12-09T08:14:13,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bd02bc1c46ee42c3ad71f25c1e0b9ac8, entries=200, sequenceid=84, filesize=14.0 K 2024-12-09T08:14:13,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/93e72dc314c24252a633c02e1f5fcdb1 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/93e72dc314c24252a633c02e1f5fcdb1 2024-12-09T08:14:13,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/93e72dc314c24252a633c02e1f5fcdb1, entries=150, sequenceid=84, filesize=11.7 K 2024-12-09T08:14:13,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/7cec363994554cbe95448910bca60b7c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/7cec363994554cbe95448910bca60b7c 2024-12-09T08:14:13,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/7cec363994554cbe95448910bca60b7c, entries=150, sequenceid=84, filesize=11.7 K 2024-12-09T08:14:13,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~203.97 KB/208864, currentSize=107.34 KB/109920 for 589b933f5d38176578d90952a67fb0a1 in 749ms, sequenceid=84, compaction requested=false 2024-12-09T08:14:13,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:13,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T08:14:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:13,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/ab0ce0c98241498daa0f07762ffbfb97 is 50, key is test_row_0/A:col10/1733732053014/Put/seqid=0 2024-12-09T08:14:13,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741852_1028 (size=12001) 2024-12-09T08:14:13,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/ab0ce0c98241498daa0f07762ffbfb97 2024-12-09T08:14:13,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T08:14:13,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:13,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732113685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732113691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732113698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732113699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732113700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/f7489ba437ac43b48f4cbc316334c53a is 50, key is test_row_0/B:col10/1733732053014/Put/seqid=0 2024-12-09T08:14:13,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741853_1029 (size=12001) 2024-12-09T08:14:13,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/f7489ba437ac43b48f4cbc316334c53a 2024-12-09T08:14:13,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/a5e6e20f1b5d4be59244812394386e8a is 50, key is test_row_0/C:col10/1733732053014/Put/seqid=0 2024-12-09T08:14:13,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741854_1030 (size=12001) 2024-12-09T08:14:13,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/a5e6e20f1b5d4be59244812394386e8a 2024-12-09T08:14:13,804 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:14:13,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/ab0ce0c98241498daa0f07762ffbfb97 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ab0ce0c98241498daa0f07762ffbfb97 2024-12-09T08:14:13,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732113812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732113812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732113812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732113815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ab0ce0c98241498daa0f07762ffbfb97, entries=150, sequenceid=107, filesize=11.7 K 2024-12-09T08:14:13,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/f7489ba437ac43b48f4cbc316334c53a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f7489ba437ac43b48f4cbc316334c53a 2024-12-09T08:14:13,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732113822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:13,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f7489ba437ac43b48f4cbc316334c53a, entries=150, sequenceid=107, filesize=11.7 K 2024-12-09T08:14:13,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/a5e6e20f1b5d4be59244812394386e8a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a5e6e20f1b5d4be59244812394386e8a 2024-12-09T08:14:13,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T08:14:13,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:13,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a5e6e20f1b5d4be59244812394386e8a, entries=150, sequenceid=107, filesize=11.7 K 2024-12-09T08:14:13,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 589b933f5d38176578d90952a67fb0a1 in 203ms, sequenceid=107, compaction requested=true 2024-12-09T08:14:13,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:13,861 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:13,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:13,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:13,862 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:13,864 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:13,864 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:13,864 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:13,864 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/A is initiating minor compaction (all files) 2024-12-09T08:14:13,864 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,864 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/A in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:13,864 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1cceb52d8ec24e3aad586029082dd846, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/93e72dc314c24252a633c02e1f5fcdb1, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f7489ba437ac43b48f4cbc316334c53a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=35.3 K 2024-12-09T08:14:13,864 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7d7673ce2efe4e90a58b562d9215a65a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bd02bc1c46ee42c3ad71f25c1e0b9ac8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ab0ce0c98241498daa0f07762ffbfb97] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=37.5 K 2024-12-09T08:14:13,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:13,865 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cceb52d8ec24e3aad586029082dd846, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733732051779 2024-12-09T08:14:13,866 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d7673ce2efe4e90a58b562d9215a65a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733732051779 2024-12-09T08:14:13,866 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 93e72dc314c24252a633c02e1f5fcdb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733732052243 2024-12-09T08:14:13,867 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting f7489ba437ac43b48f4cbc316334c53a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733732053001 2024-12-09T08:14:13,868 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd02bc1c46ee42c3ad71f25c1e0b9ac8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733732052243 2024-12-09T08:14:13,870 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab0ce0c98241498daa0f07762ffbfb97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733732053001 2024-12-09T08:14:13,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:13,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:13,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:13,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T08:14:13,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T08:14:13,891 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:13,892 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/f44ce88086294119b100e0fe5684ba74 is 50, key is test_row_0/B:col10/1733732053014/Put/seqid=0 2024-12-09T08:14:13,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T08:14:13,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T08:14:13,894 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T08:14:13,894 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T08:14:13,896 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T08:14:13,896 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-09T08:14:13,898 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:13,899 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/7bbbee170da446bfb5c51851496b4f4d is 50, key is test_row_0/A:col10/1733732053014/Put/seqid=0 2024-12-09T08:14:13,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741855_1031 (size=12207) 2024-12-09T08:14:13,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741856_1032 (size=12207) 2024-12-09T08:14:14,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T08:14:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:14,012 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T08:14:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/032cbfe696b043ffb013cd33bc302df3 is 50, key is test_row_0/A:col10/1733732053697/Put/seqid=0 2024-12-09T08:14:14,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:14,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741857_1033 (size=12001) 2024-12-09T08:14:14,037 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/032cbfe696b043ffb013cd33bc302df3 2024-12-09T08:14:14,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/95aed44bf95c439185712790dc15dced is 50, key is test_row_0/B:col10/1733732053697/Put/seqid=0 2024-12-09T08:14:14,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741858_1034 (size=12001) 2024-12-09T08:14:14,079 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/95aed44bf95c439185712790dc15dced 2024-12-09T08:14:14,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732054053, max timestamp=1733732054071Num uniques 859; 2024-12-09T08:14:14,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:14:14,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732054053, max timestamp=1733732054083Num uniques 859; 2024-12-09T08:14:14,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:14:14,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733732054053, max timestamp=1733732054071Num uniques 859; 2024-12-09T08:14:14,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T08:14:14,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/96f15294367a4dc2a106b347d1b93174 is 50, key is test_row_0/C:col10/1733732053697/Put/seqid=0 2024-12-09T08:14:14,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741859_1035 (size=12001) 2024-12-09T08:14:14,108 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/96f15294367a4dc2a106b347d1b93174 2024-12-09T08:14:14,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/032cbfe696b043ffb013cd33bc302df3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/032cbfe696b043ffb013cd33bc302df3 2024-12-09T08:14:14,128 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/032cbfe696b043ffb013cd33bc302df3, entries=150, sequenceid=120, filesize=11.7 K 2024-12-09T08:14:14,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/95aed44bf95c439185712790dc15dced as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/95aed44bf95c439185712790dc15dced 2024-12-09T08:14:14,140 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/95aed44bf95c439185712790dc15dced, entries=150, sequenceid=120, filesize=11.7 K 2024-12-09T08:14:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/96f15294367a4dc2a106b347d1b93174 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/96f15294367a4dc2a106b347d1b93174 2024-12-09T08:14:14,155 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/96f15294367a4dc2a106b347d1b93174, entries=150, sequenceid=120, filesize=11.7 K 2024-12-09T08:14:14,157 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=53.28 KB/54555 for 589b933f5d38176578d90952a67fb0a1 in 144ms, sequenceid=120, compaction requested=true 2024-12-09T08:14:14,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:14,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:14,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-09T08:14:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-09T08:14:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:14,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=59.99 KB heapSize=164.93 KB 2024-12-09T08:14:14,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-09T08:14:14,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8600 sec 2024-12-09T08:14:14,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8720 sec 2024-12-09T08:14:14,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:14,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:14,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:14,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:14,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:14,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:14,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/0bd3d3072fc8477a83b1397cf69f29b5 is 50, key is test_row_0/A:col10/1733732054160/Put/seqid=0 2024-12-09T08:14:14,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741860_1036 (size=12151) 2024-12-09T08:14:14,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/0bd3d3072fc8477a83b1397cf69f29b5 2024-12-09T08:14:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732114213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732054200, max timestamp=1733732054226Num uniques 859; 2024-12-09T08:14:14,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733732054200, max timestamp=1733732054226Num uniques 859; 2024-12-09T08:14:14,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:14:14,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:14:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/f3bfce0540fb4dffbbf001e4c77578f5 is 50, key is test_row_0/B:col10/1733732054160/Put/seqid=0 2024-12-09T08:14:14,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732054200, max timestamp=1733732054226Num uniques 859; 2024-12-09T08:14:14,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:14:14,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741861_1037 (size=12151) 2024-12-09T08:14:14,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.97 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/f3bfce0540fb4dffbbf001e4c77578f5 2024-12-09T08:14:14,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732114263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732114269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732114271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732114274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/2ac9d506e36f4b379ed0834ce1e250e5 is 50, key is test_row_0/C:col10/1733732054160/Put/seqid=0 2024-12-09T08:14:14,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741862_1038 (size=12151) 2024-12-09T08:14:14,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/2ac9d506e36f4b379ed0834ce1e250e5 2024-12-09T08:14:14,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/0bd3d3072fc8477a83b1397cf69f29b5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/0bd3d3072fc8477a83b1397cf69f29b5 2024-12-09T08:14:14,335 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/f44ce88086294119b100e0fe5684ba74 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f44ce88086294119b100e0fe5684ba74 2024-12-09T08:14:14,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/0bd3d3072fc8477a83b1397cf69f29b5, entries=150, sequenceid=147, filesize=11.9 K 2024-12-09T08:14:14,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/f3bfce0540fb4dffbbf001e4c77578f5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f3bfce0540fb4dffbbf001e4c77578f5 2024-12-09T08:14:14,341 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/7bbbee170da446bfb5c51851496b4f4d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7bbbee170da446bfb5c51851496b4f4d 2024-12-09T08:14:14,350 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into f44ce88086294119b100e0fe5684ba74(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:14,351 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:14,351 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=13, startTime=1733732053861; duration=0sec 2024-12-09T08:14:14,351 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:14,351 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:14,351 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:14:14,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f3bfce0540fb4dffbbf001e4c77578f5, entries=150, sequenceid=147, filesize=11.9 K 2024-12-09T08:14:14,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732114341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,358 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:14:14,358 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:14,359 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:14,359 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/6abc9fe9495d4ddaabaff11bc3e689d2, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/7cec363994554cbe95448910bca60b7c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a5e6e20f1b5d4be59244812394386e8a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/96f15294367a4dc2a106b347d1b93174] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=47.0 K 2024-12-09T08:14:14,359 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/A of 589b933f5d38176578d90952a67fb0a1 into 7bbbee170da446bfb5c51851496b4f4d(size=11.9 K), total size for store is 35.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:14,359 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:14,359 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/A, priority=13, startTime=1733732053861; duration=0sec 2024-12-09T08:14:14,360 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:14,360 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:14,365 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6abc9fe9495d4ddaabaff11bc3e689d2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733732051779 2024-12-09T08:14:14,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/2ac9d506e36f4b379ed0834ce1e250e5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/2ac9d506e36f4b379ed0834ce1e250e5 2024-12-09T08:14:14,369 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cec363994554cbe95448910bca60b7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733732052243 2024-12-09T08:14:14,370 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e6e20f1b5d4be59244812394386e8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733732053001 2024-12-09T08:14:14,371 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 96f15294367a4dc2a106b347d1b93174, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733732053678 2024-12-09T08:14:14,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/2ac9d506e36f4b379ed0834ce1e250e5, entries=150, sequenceid=147, filesize=11.9 K 2024-12-09T08:14:14,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~66.69 KB/68295, heapSize ~184.80 KB/189240, currentSize=120.76 KB/123660 for 589b933f5d38176578d90952a67fb0a1 in 218ms, sequenceid=147, compaction requested=true 2024-12-09T08:14:14,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:14,379 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:14,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:14,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:14,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:14,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:14,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:14,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T08:14:14,382 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:14,382 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/A is initiating minor compaction (all files) 2024-12-09T08:14:14,382 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/A in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:14,382 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7bbbee170da446bfb5c51851496b4f4d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/032cbfe696b043ffb013cd33bc302df3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/0bd3d3072fc8477a83b1397cf69f29b5] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=35.5 K 2024-12-09T08:14:14,383 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bbbee170da446bfb5c51851496b4f4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733732053001 2024-12-09T08:14:14,384 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 032cbfe696b043ffb013cd33bc302df3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733732053678 2024-12-09T08:14:14,385 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bd3d3072fc8477a83b1397cf69f29b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733732054053 2024-12-09T08:14:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:14,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-09T08:14:14,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:14,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:14,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:14,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:14,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:14,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:14,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/cd21012497ce468ea83cac93c271df5a is 50, key is test_row_0/A:col10/1733732054256/Put/seqid=0 2024-12-09T08:14:14,409 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:14,410 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/653bf656e5be4d6b86aaf55928a47ae4 is 50, key is test_row_0/C:col10/1733732053697/Put/seqid=0 2024-12-09T08:14:14,414 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#A#compaction#28 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:14,415 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/10c40b62a81240ae89ae7bc50b7bb57e is 50, key is test_row_0/A:col10/1733732054160/Put/seqid=0 2024-12-09T08:14:14,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741863_1039 (size=12241) 2024-12-09T08:14:14,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732114425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732114432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732114434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732114434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,440 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/653bf656e5be4d6b86aaf55928a47ae4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/653bf656e5be4d6b86aaf55928a47ae4 2024-12-09T08:14:14,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741864_1040 (size=14541) 2024-12-09T08:14:14,451 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into 653bf656e5be4d6b86aaf55928a47ae4(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:14,452 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:14,452 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=12, startTime=1733732053874; duration=0sec 2024-12-09T08:14:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T08:14:14,452 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T08:14:14,452 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:14,452 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:14,452 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:14,452 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:14,456 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-09T08:14:14,460 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:14,460 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:14,460 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:14,460 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f44ce88086294119b100e0fe5684ba74, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/95aed44bf95c439185712790dc15dced, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f3bfce0540fb4dffbbf001e4c77578f5] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=35.5 K 2024-12-09T08:14:14,462 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting f44ce88086294119b100e0fe5684ba74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733732053001 2024-12-09T08:14:14,463 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:14,463 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 95aed44bf95c439185712790dc15dced, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733732053678 2024-12-09T08:14:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T08:14:14,464 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting f3bfce0540fb4dffbbf001e4c77578f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733732054053 2024-12-09T08:14:14,464 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:14,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:14,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741865_1041 (size=12459) 2024-12-09T08:14:14,483 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/10c40b62a81240ae89ae7bc50b7bb57e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/10c40b62a81240ae89ae7bc50b7bb57e 2024-12-09T08:14:14,494 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/A of 589b933f5d38176578d90952a67fb0a1 into 10c40b62a81240ae89ae7bc50b7bb57e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:14,494 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:14,494 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/A, priority=13, startTime=1733732054379; duration=0sec 2024-12-09T08:14:14,494 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:14,495 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:14,495 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T08:14:14,496 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T08:14:14,497 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T08:14:14,497 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. because compaction request was cancelled 2024-12-09T08:14:14,497 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:14,504 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#29 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:14,504 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/7ce78dc04658459c8d0b128d5afe2930 is 50, key is test_row_0/B:col10/1733732054160/Put/seqid=0 2024-12-09T08:14:14,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741866_1042 (size=12459) 2024-12-09T08:14:14,539 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/7ce78dc04658459c8d0b128d5afe2930 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/7ce78dc04658459c8d0b128d5afe2930 2024-12-09T08:14:14,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732114542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732114542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732114542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,553 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into 7ce78dc04658459c8d0b128d5afe2930(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:14,553 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:14,553 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=13, startTime=1733732054380; duration=0sec 2024-12-09T08:14:14,553 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:14,553 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:14,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732114552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732114561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T08:14:14,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T08:14:14,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:14,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:14,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:14,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:14,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:14,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732114751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732114752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T08:14:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:14,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:14,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732114775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T08:14:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732114752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/cd21012497ce468ea83cac93c271df5a 2024-12-09T08:14:14,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/862f467d100e408ea1ff9d82075a97da is 50, key is test_row_0/B:col10/1733732054256/Put/seqid=0 2024-12-09T08:14:14,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732114872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:14,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:14,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741867_1043 (size=12151) 2024-12-09T08:14:14,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/862f467d100e408ea1ff9d82075a97da 2024-12-09T08:14:14,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/a52b60ecf86242949315c3cd9d7c06d1 is 50, key is test_row_0/C:col10/1733732054256/Put/seqid=0 2024-12-09T08:14:14,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T08:14:14,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:14,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:14,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:14,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:14,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:14,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741868_1044 (size=12151) 2024-12-09T08:14:15,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732115062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732115081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T08:14:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:15,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:15,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T08:14:15,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732115092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732115101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T08:14:15,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:15,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:15,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:15,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:15,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:15,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/a52b60ecf86242949315c3cd9d7c06d1 2024-12-09T08:14:15,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/cd21012497ce468ea83cac93c271df5a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/cd21012497ce468ea83cac93c271df5a 2024-12-09T08:14:15,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/cd21012497ce468ea83cac93c271df5a, entries=200, sequenceid=184, filesize=14.2 K 2024-12-09T08:14:15,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/862f467d100e408ea1ff9d82075a97da as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/862f467d100e408ea1ff9d82075a97da 2024-12-09T08:14:15,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732115382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/862f467d100e408ea1ff9d82075a97da, entries=150, sequenceid=184, filesize=11.9 K 2024-12-09T08:14:15,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/a52b60ecf86242949315c3cd9d7c06d1 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a52b60ecf86242949315c3cd9d7c06d1 2024-12-09T08:14:15,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T08:14:15,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:15,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:15,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:15,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:15,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:15,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a52b60ecf86242949315c3cd9d7c06d1, entries=150, sequenceid=184, filesize=11.9 K 2024-12-09T08:14:15,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 589b933f5d38176578d90952a67fb0a1 in 1019ms, sequenceid=184, compaction requested=true 2024-12-09T08:14:15,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:15,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:15,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:15,409 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T08:14:15,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:15,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:15,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:15,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T08:14:15,409 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:15,411 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T08:14:15,411 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T08:14:15,411 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. because compaction request was cancelled 2024-12-09T08:14:15,412 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:15,412 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T08:14:15,414 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:15,414 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:15,414 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:15,414 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/653bf656e5be4d6b86aaf55928a47ae4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/2ac9d506e36f4b379ed0834ce1e250e5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a52b60ecf86242949315c3cd9d7c06d1] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=35.7 K 2024-12-09T08:14:15,415 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T08:14:15,415 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T08:14:15,415 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. because compaction request was cancelled 2024-12-09T08:14:15,415 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:15,415 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 653bf656e5be4d6b86aaf55928a47ae4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733732053678 2024-12-09T08:14:15,416 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ac9d506e36f4b379ed0834ce1e250e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733732054053 2024-12-09T08:14:15,416 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a52b60ecf86242949315c3cd9d7c06d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733732054256 2024-12-09T08:14:15,442 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#32 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:15,443 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/b450a3086aca4b2b96cd43d9f2237fcd is 50, key is test_row_0/C:col10/1733732054256/Put/seqid=0 2024-12-09T08:14:15,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741869_1045 (size=12493) 2024-12-09T08:14:15,470 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/b450a3086aca4b2b96cd43d9f2237fcd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/b450a3086aca4b2b96cd43d9f2237fcd 2024-12-09T08:14:15,484 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into b450a3086aca4b2b96cd43d9f2237fcd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:15,485 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:15,485 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=13, startTime=1733732055409; duration=0sec 2024-12-09T08:14:15,485 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:15,485 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:15,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T08:14:15,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:15,560 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:14:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:15,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/613e3019dd3443978fc211c272761fce is 50, key is test_row_0/A:col10/1733732054430/Put/seqid=0 2024-12-09T08:14:15,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:15,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741870_1046 (size=9757) 2024-12-09T08:14:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T08:14:15,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732055608, max timestamp=1733732055637Num uniques 859; 2024-12-09T08:14:15,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:14:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732115632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732115637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733732055608, max timestamp=1733732055637Num uniques 859; 2024-12-09T08:14:15,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T08:14:15,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732055608, max timestamp=1733732055637Num uniques 859; 2024-12-09T08:14:15,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:14:15,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732115691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732115692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732115751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732115751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732115801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732115801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732115961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732115962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:15,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:15,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:15,985 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/613e3019dd3443978fc211c272761fce 2024-12-09T08:14:15,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/113451ed3a4b4719beccd75a5763c249 is 50, key is test_row_0/B:col10/1733732054430/Put/seqid=0 2024-12-09T08:14:16,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741871_1047 (size=9757) 2024-12-09T08:14:16,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732116011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732116011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:16,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732116271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732116272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:16,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:16,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732116321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732116322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:16,405 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/113451ed3a4b4719beccd75a5763c249 2024-12-09T08:14:16,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732116401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:16,410 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:16,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/048959e9475249f68b321867abe51398 is 50, key is test_row_0/C:col10/1733732054430/Put/seqid=0 2024-12-09T08:14:16,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741872_1048 (size=9757) 2024-12-09T08:14:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T08:14:16,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732116782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732116782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:16,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732116832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:16,834 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/048959e9475249f68b321867abe51398 2024-12-09T08:14:16,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/613e3019dd3443978fc211c272761fce as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/613e3019dd3443978fc211c272761fce 2024-12-09T08:14:16,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732116843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:16,853 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/613e3019dd3443978fc211c272761fce, entries=100, sequenceid=199, filesize=9.5 K 2024-12-09T08:14:16,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/113451ed3a4b4719beccd75a5763c249 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/113451ed3a4b4719beccd75a5763c249 2024-12-09T08:14:16,863 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/113451ed3a4b4719beccd75a5763c249, entries=100, sequenceid=199, filesize=9.5 K 2024-12-09T08:14:16,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/048959e9475249f68b321867abe51398 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/048959e9475249f68b321867abe51398 2024-12-09T08:14:16,877 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/048959e9475249f68b321867abe51398, entries=100, sequenceid=199, filesize=9.5 K 2024-12-09T08:14:16,879 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 589b933f5d38176578d90952a67fb0a1 in 1318ms, sequenceid=199, compaction requested=true 2024-12-09T08:14:16,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:16,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:16,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-09T08:14:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-09T08:14:16,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-09T08:14:16,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4170 sec 2024-12-09T08:14:16,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.4320 sec 2024-12-09T08:14:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:17,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=140.49 KB heapSize=404.70 KB 2024-12-09T08:14:17,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:17,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T08:14:17,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:17,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T08:14:17,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:17,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T08:14:17,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/d41ae45e4abd489e8f896a0173c4e577 is 50, key is test_row_0/A:col10/1733732055688/Put/seqid=0 2024-12-09T08:14:17,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741873_1049 (size=16931) 2024-12-09T08:14:17,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/d41ae45e4abd489e8f896a0173c4e577 2024-12-09T08:14:17,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732117822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:17,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732117824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:17,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/de74ab8169344e46bcbb018bb40b16ee is 50, key is test_row_0/B:col10/1733732055688/Put/seqid=0 2024-12-09T08:14:17,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732117841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:17,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741874_1050 (size=12151) 2024-12-09T08:14:17,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/de74ab8169344e46bcbb018bb40b16ee 2024-12-09T08:14:17,843 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:17,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/64ea61bf5ec9414eaab673903a99fc3f is 50, key is test_row_0/C:col10/1733732055688/Put/seqid=0 2024-12-09T08:14:17,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732117862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:17,864 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:17,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741875_1051 (size=12151) 2024-12-09T08:14:17,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732117931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:17,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732117931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:18,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732118141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732118141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:18,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/64ea61bf5ec9414eaab673903a99fc3f 2024-12-09T08:14:18,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/d41ae45e4abd489e8f896a0173c4e577 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/d41ae45e4abd489e8f896a0173c4e577 2024-12-09T08:14:18,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/d41ae45e4abd489e8f896a0173c4e577, entries=250, sequenceid=237, filesize=16.5 K 2024-12-09T08:14:18,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/de74ab8169344e46bcbb018bb40b16ee as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/de74ab8169344e46bcbb018bb40b16ee 2024-12-09T08:14:18,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/de74ab8169344e46bcbb018bb40b16ee, entries=150, sequenceid=237, filesize=11.9 K 2024-12-09T08:14:18,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/64ea61bf5ec9414eaab673903a99fc3f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/64ea61bf5ec9414eaab673903a99fc3f 2024-12-09T08:14:18,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/64ea61bf5ec9414eaab673903a99fc3f, entries=150, sequenceid=237, filesize=11.9 K 2024-12-09T08:14:18,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.49 KB/143865, heapSize ~404.65 KB/414360, currentSize=40.25 KB/41220 for 589b933f5d38176578d90952a67fb0a1 in 515ms, sequenceid=237, compaction requested=true 2024-12-09T08:14:18,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:18,320 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:14:18,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:18,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:18,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:18,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:18,321 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:14:18,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:18,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:18,322 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:14:18,323 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53688 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:14:18,323 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/A is initiating minor compaction (all files) 2024-12-09T08:14:18,323 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/A in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:18,323 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/10c40b62a81240ae89ae7bc50b7bb57e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/cd21012497ce468ea83cac93c271df5a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/613e3019dd3443978fc211c272761fce, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/d41ae45e4abd489e8f896a0173c4e577] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=52.4 K 2024-12-09T08:14:18,324 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:18,324 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:18,324 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/7ce78dc04658459c8d0b128d5afe2930, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/862f467d100e408ea1ff9d82075a97da, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/113451ed3a4b4719beccd75a5763c249, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/de74ab8169344e46bcbb018bb40b16ee] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=45.4 K 2024-12-09T08:14:18,324 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10c40b62a81240ae89ae7bc50b7bb57e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733732054053 2024-12-09T08:14:18,325 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd21012497ce468ea83cac93c271df5a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733732054256 2024-12-09T08:14:18,325 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ce78dc04658459c8d0b128d5afe2930, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733732054053 2024-12-09T08:14:18,325 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 613e3019dd3443978fc211c272761fce, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733732054430 2024-12-09T08:14:18,325 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 862f467d100e408ea1ff9d82075a97da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733732054256 2024-12-09T08:14:18,326 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d41ae45e4abd489e8f896a0173c4e577, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733732055665 2024-12-09T08:14:18,326 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 113451ed3a4b4719beccd75a5763c249, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733732054430 2024-12-09T08:14:18,327 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting de74ab8169344e46bcbb018bb40b16ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733732055665 2024-12-09T08:14:18,342 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:18,343 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/6cf16d6554e54c1e9cd2db5ffe021aef is 50, key is test_row_0/A:col10/1733732055688/Put/seqid=0 2024-12-09T08:14:18,347 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:18,348 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/df72b8e0397f4169b5a14acaaac2f1ca is 50, key is test_row_0/B:col10/1733732055688/Put/seqid=0 2024-12-09T08:14:18,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741876_1052 (size=12595) 2024-12-09T08:14:18,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741877_1053 (size=12595) 2024-12-09T08:14:18,385 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/6cf16d6554e54c1e9cd2db5ffe021aef as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/6cf16d6554e54c1e9cd2db5ffe021aef 2024-12-09T08:14:18,392 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/df72b8e0397f4169b5a14acaaac2f1ca as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/df72b8e0397f4169b5a14acaaac2f1ca 2024-12-09T08:14:18,442 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into df72b8e0397f4169b5a14acaaac2f1ca(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:18,443 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:18,443 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=12, startTime=1733732058320; duration=0sec 2024-12-09T08:14:18,443 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:18,443 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:18,443 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:18,444 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/A of 589b933f5d38176578d90952a67fb0a1 into 6cf16d6554e54c1e9cd2db5ffe021aef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:18,444 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:18,444 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/A, priority=12, startTime=1733732058320; duration=0sec 2024-12-09T08:14:18,444 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:18,444 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:18,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:14:18,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:18,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:18,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:18,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:18,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:18,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:18,449 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:18,449 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:18,449 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:18,449 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/b450a3086aca4b2b96cd43d9f2237fcd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/048959e9475249f68b321867abe51398, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/64ea61bf5ec9414eaab673903a99fc3f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=33.6 K 2024-12-09T08:14:18,450 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting b450a3086aca4b2b96cd43d9f2237fcd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733732054256 2024-12-09T08:14:18,451 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 048959e9475249f68b321867abe51398, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733732054430 2024-12-09T08:14:18,453 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ea61bf5ec9414eaab673903a99fc3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733732055665 2024-12-09T08:14:18,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/8710984f930142cfb3320927936ec8c7 is 50, key is test_row_0/A:col10/1733732057810/Put/seqid=0 2024-12-09T08:14:18,473 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:18,474 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/2d8ac1ebe1f34567882b50e6644b29b1 is 50, key is test_row_0/C:col10/1733732055688/Put/seqid=0 2024-12-09T08:14:18,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733732058471, max timestamp=1733732058498Num uniques 859; 2024-12-09T08:14:18,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T08:14:18,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733732058471, max timestamp=1733732058498Num uniques 859; 2024-12-09T08:14:18,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T08:14:18,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733732058471, max timestamp=1733732058498Num uniques 859; 2024-12-09T08:14:18,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T08:14:18,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741878_1054 (size=14541) 2024-12-09T08:14:18,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/8710984f930142cfb3320927936ec8c7 2024-12-09T08:14:18,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741879_1055 (size=12595) 2024-12-09T08:14:18,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/1d6b675395f745819f34b87a4ad3324d is 50, key is test_row_0/B:col10/1733732057810/Put/seqid=0 2024-12-09T08:14:18,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741880_1056 (size=12151) 2024-12-09T08:14:18,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732118545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732118545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:18,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732118551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T08:14:18,632 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:18,632 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-09T08:14:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T08:14:18,635 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:18,635 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:18,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:18,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732118661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:18,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:18,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732118661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732118661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T08:14:18,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T08:14:18,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:18,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:18,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:18,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:18,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:18,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732118871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:18,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732118871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:18,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732118871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:18,920 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/2d8ac1ebe1f34567882b50e6644b29b1 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/2d8ac1ebe1f34567882b50e6644b29b1 2024-12-09T08:14:18,932 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into 2d8ac1ebe1f34567882b50e6644b29b1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:18,932 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:18,932 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=13, startTime=1733732058321; duration=0sec 2024-12-09T08:14:18,932 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:18,932 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:18,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T08:14:18,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:18,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:18,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/1d6b675395f745819f34b87a4ad3324d 2024-12-09T08:14:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T08:14:18,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/67c223cf29dd4beebbc7c9d08994999a is 50, key is test_row_0/C:col10/1733732057810/Put/seqid=0 2024-12-09T08:14:18,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741881_1057 (size=12151) 2024-12-09T08:14:18,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/67c223cf29dd4beebbc7c9d08994999a 2024-12-09T08:14:18,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/8710984f930142cfb3320927936ec8c7 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/8710984f930142cfb3320927936ec8c7 2024-12-09T08:14:18,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/8710984f930142cfb3320927936ec8c7, entries=200, sequenceid=249, filesize=14.2 K 2024-12-09T08:14:18,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/1d6b675395f745819f34b87a4ad3324d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1d6b675395f745819f34b87a4ad3324d 2024-12-09T08:14:19,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1d6b675395f745819f34b87a4ad3324d, entries=150, sequenceid=249, filesize=11.9 K 2024-12-09T08:14:19,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/67c223cf29dd4beebbc7c9d08994999a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/67c223cf29dd4beebbc7c9d08994999a 2024-12-09T08:14:19,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/67c223cf29dd4beebbc7c9d08994999a, entries=150, sequenceid=249, filesize=11.9 K 2024-12-09T08:14:19,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 589b933f5d38176578d90952a67fb0a1 in 567ms, sequenceid=249, compaction requested=false 2024-12-09T08:14:19,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:19,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T08:14:19,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:19,097 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-09T08:14:19,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:19,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:19,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:19,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:19,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:19,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:19,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/39849e29decd425896a2d192cd53fead is 50, key is test_row_0/A:col10/1733732058539/Put/seqid=0 2024-12-09T08:14:19,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741882_1058 (size=12301) 2024-12-09T08:14:19,118 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/39849e29decd425896a2d192cd53fead 2024-12-09T08:14:19,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/6f289659aea04267b404b7cf360ff969 is 50, key is test_row_0/B:col10/1733732058539/Put/seqid=0 2024-12-09T08:14:19,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741883_1059 (size=12301) 2024-12-09T08:14:19,137 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/6f289659aea04267b404b7cf360ff969 2024-12-09T08:14:19,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/deb1e7cf2c904138be43ddce51d956d4 is 50, key is test_row_0/C:col10/1733732058539/Put/seqid=0 2024-12-09T08:14:19,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741884_1060 (size=12301) 2024-12-09T08:14:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:19,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:19,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732119192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732119197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732119197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T08:14:19,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732119301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732119311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732119311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732119511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732119521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732119521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,566 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/deb1e7cf2c904138be43ddce51d956d4 2024-12-09T08:14:19,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/39849e29decd425896a2d192cd53fead as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/39849e29decd425896a2d192cd53fead 2024-12-09T08:14:19,579 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/39849e29decd425896a2d192cd53fead, entries=150, sequenceid=285, filesize=12.0 K 2024-12-09T08:14:19,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/6f289659aea04267b404b7cf360ff969 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/6f289659aea04267b404b7cf360ff969 2024-12-09T08:14:19,587 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/6f289659aea04267b404b7cf360ff969, entries=150, sequenceid=285, filesize=12.0 K 2024-12-09T08:14:19,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/deb1e7cf2c904138be43ddce51d956d4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/deb1e7cf2c904138be43ddce51d956d4 2024-12-09T08:14:19,594 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/deb1e7cf2c904138be43ddce51d956d4, entries=150, sequenceid=285, filesize=12.0 K 2024-12-09T08:14:19,595 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for 589b933f5d38176578d90952a67fb0a1 in 498ms, sequenceid=285, compaction requested=true 2024-12-09T08:14:19,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:19,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:19,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-09T08:14:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-09T08:14:19,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T08:14:19,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 962 msec 2024-12-09T08:14:19,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 971 msec 2024-12-09T08:14:19,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T08:14:19,771 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:19,772 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-09T08:14:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T08:14:19,774 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:19,775 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:19,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:19,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T08:14:19,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:19,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:19,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:19,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:19,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:19,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:19,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/966dd50f0a034f2f833b48a432b4320b is 50, key is test_row_0/A:col10/1733732059196/Put/seqid=0 2024-12-09T08:14:19,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741885_1061 (size=14741) 2024-12-09T08:14:19,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/966dd50f0a034f2f833b48a432b4320b 2024-12-09T08:14:19,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/6f7495d3d4b64456b4dd40d55944b471 is 50, key is test_row_0/B:col10/1733732059196/Put/seqid=0 2024-12-09T08:14:19,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741886_1062 (size=12301) 2024-12-09T08:14:19,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732119862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732119868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,871 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732119871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732119871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T08:14:19,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732119891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,893 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T08:14:19,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:19,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:19,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:19,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:19,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:19,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732119981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732119981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732119982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T08:14:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:20,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T08:14:20,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732120191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732120191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732120192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T08:14:20,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:20,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:20,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:20,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:20,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:20,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/6f7495d3d4b64456b4dd40d55944b471 2024-12-09T08:14:20,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/647bb4ac2c0e4f18a2b7e69bb17b63f8 is 50, key is test_row_0/C:col10/1733732059196/Put/seqid=0 2024-12-09T08:14:20,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741887_1063 (size=12301) 2024-12-09T08:14:20,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/647bb4ac2c0e4f18a2b7e69bb17b63f8 2024-12-09T08:14:20,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/966dd50f0a034f2f833b48a432b4320b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/966dd50f0a034f2f833b48a432b4320b 2024-12-09T08:14:20,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/966dd50f0a034f2f833b48a432b4320b, entries=200, sequenceid=297, filesize=14.4 K 2024-12-09T08:14:20,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/6f7495d3d4b64456b4dd40d55944b471 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/6f7495d3d4b64456b4dd40d55944b471 2024-12-09T08:14:20,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/6f7495d3d4b64456b4dd40d55944b471, entries=150, sequenceid=297, filesize=12.0 K 2024-12-09T08:14:20,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/647bb4ac2c0e4f18a2b7e69bb17b63f8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/647bb4ac2c0e4f18a2b7e69bb17b63f8 2024-12-09T08:14:20,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/647bb4ac2c0e4f18a2b7e69bb17b63f8, entries=150, sequenceid=297, filesize=12.0 K 2024-12-09T08:14:20,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 589b933f5d38176578d90952a67fb0a1 in 494ms, sequenceid=297, compaction requested=true 2024-12-09T08:14:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:20,319 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:14:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:20,319 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:14:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:20,320 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54178 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:14:20,320 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:14:20,320 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/A is initiating minor compaction (all files) 2024-12-09T08:14:20,320 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:20,320 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/A in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:20,320 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:20,320 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/6cf16d6554e54c1e9cd2db5ffe021aef, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/8710984f930142cfb3320927936ec8c7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/39849e29decd425896a2d192cd53fead, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/966dd50f0a034f2f833b48a432b4320b] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=52.9 K 2024-12-09T08:14:20,320 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/df72b8e0397f4169b5a14acaaac2f1ca, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1d6b675395f745819f34b87a4ad3324d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/6f289659aea04267b404b7cf360ff969, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/6f7495d3d4b64456b4dd40d55944b471] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=48.2 K 2024-12-09T08:14:20,321 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting df72b8e0397f4169b5a14acaaac2f1ca, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733732055665 2024-12-09T08:14:20,321 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cf16d6554e54c1e9cd2db5ffe021aef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733732055665 2024-12-09T08:14:20,321 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d6b675395f745819f34b87a4ad3324d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733732057810 2024-12-09T08:14:20,321 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8710984f930142cfb3320927936ec8c7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733732057810 2024-12-09T08:14:20,322 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f289659aea04267b404b7cf360ff969, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733732058539 2024-12-09T08:14:20,322 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f7495d3d4b64456b4dd40d55944b471, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733732059195 2024-12-09T08:14:20,322 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39849e29decd425896a2d192cd53fead, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733732058539 2024-12-09T08:14:20,322 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 966dd50f0a034f2f833b48a432b4320b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733732059190 2024-12-09T08:14:20,334 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:20,334 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:20,335 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/3fa719f12c524cebb5da2bf430ddb587 is 50, key is test_row_0/B:col10/1733732059196/Put/seqid=0 2024-12-09T08:14:20,335 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/a1adfa968a9146158956430f27f5b188 is 50, key is test_row_0/A:col10/1733732059196/Put/seqid=0 2024-12-09T08:14:20,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741889_1065 (size=12881) 2024-12-09T08:14:20,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741888_1064 (size=12881) 2024-12-09T08:14:20,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T08:14:20,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:20,392 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T08:14:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:20,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:20,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/ba0bbbb1a27247f7a4abfdb156fd11f0 is 50, key is test_row_0/A:col10/1733732059866/Put/seqid=0 2024-12-09T08:14:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T08:14:20,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741890_1066 (size=12301) 2024-12-09T08:14:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:20,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:20,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732120518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:20,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732120524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732120525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732120631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:20,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732120631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732120641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,753 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/a1adfa968a9146158956430f27f5b188 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a1adfa968a9146158956430f27f5b188 2024-12-09T08:14:20,753 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/3fa719f12c524cebb5da2bf430ddb587 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/3fa719f12c524cebb5da2bf430ddb587 2024-12-09T08:14:20,761 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into 3fa719f12c524cebb5da2bf430ddb587(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:20,761 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:20,761 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=12, startTime=1733732060319; duration=0sec 2024-12-09T08:14:20,762 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:20,762 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:20,762 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:14:20,763 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/A of 589b933f5d38176578d90952a67fb0a1 into a1adfa968a9146158956430f27f5b188(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:20,763 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:20,763 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:14:20,763 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/A, priority=12, startTime=1733732060319; duration=0sec 2024-12-09T08:14:20,764 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:20,764 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:20,764 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:20,764 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:20,764 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/2d8ac1ebe1f34567882b50e6644b29b1, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/67c223cf29dd4beebbc7c9d08994999a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/deb1e7cf2c904138be43ddce51d956d4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/647bb4ac2c0e4f18a2b7e69bb17b63f8] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=48.2 K 2024-12-09T08:14:20,764 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d8ac1ebe1f34567882b50e6644b29b1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733732055665 2024-12-09T08:14:20,765 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c223cf29dd4beebbc7c9d08994999a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733732057810 2024-12-09T08:14:20,766 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting deb1e7cf2c904138be43ddce51d956d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733732058539 2024-12-09T08:14:20,766 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 647bb4ac2c0e4f18a2b7e69bb17b63f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733732059195 2024-12-09T08:14:20,777 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#54 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:20,781 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/88e4952aedaa4f2f8b47c3be8d0cef92 is 50, key is test_row_0/C:col10/1733732059196/Put/seqid=0 2024-12-09T08:14:20,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741891_1067 (size=12881) 2024-12-09T08:14:20,807 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/ba0bbbb1a27247f7a4abfdb156fd11f0 2024-12-09T08:14:20,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/53377803a78f42a8b06ad9230c99b2eb is 50, key is test_row_0/B:col10/1733732059866/Put/seqid=0 2024-12-09T08:14:20,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741892_1068 (size=12301) 2024-12-09T08:14:20,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:20,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732120841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732120841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732120851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T08:14:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732121151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:21,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732121151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:21,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732121161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:21,196 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/88e4952aedaa4f2f8b47c3be8d0cef92 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/88e4952aedaa4f2f8b47c3be8d0cef92 2024-12-09T08:14:21,203 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into 88e4952aedaa4f2f8b47c3be8d0cef92(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:21,203 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:21,203 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=12, startTime=1733732060319; duration=0sec 2024-12-09T08:14:21,203 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:21,203 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:21,221 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/53377803a78f42a8b06ad9230c99b2eb 2024-12-09T08:14:21,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/e565bf0fe6d2495cbda0d641813a0580 is 50, key is test_row_0/C:col10/1733732059866/Put/seqid=0 2024-12-09T08:14:21,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741893_1069 (size=12301) 2024-12-09T08:14:21,637 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/e565bf0fe6d2495cbda0d641813a0580 2024-12-09T08:14:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/ba0bbbb1a27247f7a4abfdb156fd11f0 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ba0bbbb1a27247f7a4abfdb156fd11f0 2024-12-09T08:14:21,650 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ba0bbbb1a27247f7a4abfdb156fd11f0, entries=150, sequenceid=317, filesize=12.0 K 2024-12-09T08:14:21,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/53377803a78f42a8b06ad9230c99b2eb as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/53377803a78f42a8b06ad9230c99b2eb 2024-12-09T08:14:21,659 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/53377803a78f42a8b06ad9230c99b2eb, entries=150, sequenceid=317, filesize=12.0 K 2024-12-09T08:14:21,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/e565bf0fe6d2495cbda0d641813a0580 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/e565bf0fe6d2495cbda0d641813a0580 2024-12-09T08:14:21,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732121661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:21,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732121661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:21,669 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/e565bf0fe6d2495cbda0d641813a0580, entries=150, sequenceid=317, filesize=12.0 K 2024-12-09T08:14:21,670 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 589b933f5d38176578d90952a67fb0a1 in 1278ms, sequenceid=317, compaction requested=false 2024-12-09T08:14:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-09T08:14:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-09T08:14:21,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-09T08:14:21,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8970 sec 2024-12-09T08:14:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:21,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T08:14:21,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:21,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:21,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:21,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:21,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:21,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:21,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.9060 sec 2024-12-09T08:14:21,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/f920a8e9c7ad4a858a362a1650c1c501 is 50, key is test_row_0/A:col10/1733732061674/Put/seqid=0 2024-12-09T08:14:21,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741894_1070 (size=12301) 2024-12-09T08:14:21,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/f920a8e9c7ad4a858a362a1650c1c501 2024-12-09T08:14:21,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/d85ddd77b34b42c6a265b1f962b8bdfc is 50, key is test_row_0/B:col10/1733732061674/Put/seqid=0 2024-12-09T08:14:21,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741895_1071 (size=12301) 2024-12-09T08:14:21,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/d85ddd77b34b42c6a265b1f962b8bdfc 2024-12-09T08:14:21,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/8ebc702668064f8e8b60a02941b706ca is 50, key is test_row_0/C:col10/1733732061674/Put/seqid=0 2024-12-09T08:14:21,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732121764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:21,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741896_1072 (size=12301) 2024-12-09T08:14:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:21,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/8ebc702668064f8e8b60a02941b706ca 2024-12-09T08:14:21,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/f920a8e9c7ad4a858a362a1650c1c501 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/f920a8e9c7ad4a858a362a1650c1c501 2024-12-09T08:14:21,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/f920a8e9c7ad4a858a362a1650c1c501, entries=150, sequenceid=334, filesize=12.0 K 2024-12-09T08:14:21,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/d85ddd77b34b42c6a265b1f962b8bdfc as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/d85ddd77b34b42c6a265b1f962b8bdfc 2024-12-09T08:14:21,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/d85ddd77b34b42c6a265b1f962b8bdfc, entries=150, sequenceid=334, filesize=12.0 K 2024-12-09T08:14:21,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/8ebc702668064f8e8b60a02941b706ca as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ebc702668064f8e8b60a02941b706ca 2024-12-09T08:14:21,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ebc702668064f8e8b60a02941b706ca, entries=150, sequenceid=334, filesize=12.0 K 2024-12-09T08:14:21,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 589b933f5d38176578d90952a67fb0a1 in 133ms, sequenceid=334, compaction requested=true 2024-12-09T08:14:21,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:21,812 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:21,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:21,813 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:21,813 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:21,814 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/A is initiating minor compaction (all files) 2024-12-09T08:14:21,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:21,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:21,814 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/A in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:21,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:21,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:21,814 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a1adfa968a9146158956430f27f5b188, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ba0bbbb1a27247f7a4abfdb156fd11f0, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/f920a8e9c7ad4a858a362a1650c1c501] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=36.6 K 2024-12-09T08:14:21,815 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1adfa968a9146158956430f27f5b188, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733732059195 2024-12-09T08:14:21,815 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:21,815 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:21,816 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:21,816 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/3fa719f12c524cebb5da2bf430ddb587, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/53377803a78f42a8b06ad9230c99b2eb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/d85ddd77b34b42c6a265b1f962b8bdfc] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=36.6 K 2024-12-09T08:14:21,816 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fa719f12c524cebb5da2bf430ddb587, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733732059195 2024-12-09T08:14:21,817 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba0bbbb1a27247f7a4abfdb156fd11f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733732059866 2024-12-09T08:14:21,817 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 53377803a78f42a8b06ad9230c99b2eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733732059866 2024-12-09T08:14:21,818 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting f920a8e9c7ad4a858a362a1650c1c501, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733732060509 2024-12-09T08:14:21,819 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting d85ddd77b34b42c6a265b1f962b8bdfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733732060509 2024-12-09T08:14:21,835 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:21,836 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/99ba7731daec4e8780fcd68080d52cac is 50, key is test_row_0/A:col10/1733732061674/Put/seqid=0 2024-12-09T08:14:21,839 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:21,840 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/704756a15fce49b09dedd2a95b659730 is 50, key is test_row_0/B:col10/1733732061674/Put/seqid=0 2024-12-09T08:14:21,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741898_1074 (size=12983) 2024-12-09T08:14:21,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741897_1073 (size=12983) 2024-12-09T08:14:21,875 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/704756a15fce49b09dedd2a95b659730 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/704756a15fce49b09dedd2a95b659730 2024-12-09T08:14:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:21,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T08:14:21,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:21,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:21,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:21,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:21,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:21,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:21,886 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into 704756a15fce49b09dedd2a95b659730(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:21,887 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:21,887 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=13, startTime=1733732061813; duration=0sec 2024-12-09T08:14:21,887 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:21,887 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:21,887 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:21,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/e622f3285aca4ce792c90431a1154955 is 50, key is test_row_0/A:col10/1733732061872/Put/seqid=0 2024-12-09T08:14:21,891 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:21,891 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:21,891 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:21,892 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/88e4952aedaa4f2f8b47c3be8d0cef92, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/e565bf0fe6d2495cbda0d641813a0580, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ebc702668064f8e8b60a02941b706ca] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=36.6 K 2024-12-09T08:14:21,893 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 88e4952aedaa4f2f8b47c3be8d0cef92, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733732059195 2024-12-09T08:14:21,894 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting e565bf0fe6d2495cbda0d641813a0580, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733732059866 2024-12-09T08:14:21,895 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ebc702668064f8e8b60a02941b706ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733732060509 2024-12-09T08:14:21,911 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:21,912 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/4d61090df7f242afac304de805fdacf6 is 50, key is test_row_0/C:col10/1733732061674/Put/seqid=0 2024-12-09T08:14:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T08:14:21,922 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:21,922 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-09T08:14:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T08:14:21,926 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:21,927 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:21,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:21,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741899_1075 (size=14741) 2024-12-09T08:14:21,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/e622f3285aca4ce792c90431a1154955 2024-12-09T08:14:21,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732121948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:21,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/211b217f282048609bb9418b0abb8ba6 is 50, key is test_row_0/B:col10/1733732061872/Put/seqid=0 2024-12-09T08:14:21,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741900_1076 (size=12983) 2024-12-09T08:14:21,971 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/4d61090df7f242afac304de805fdacf6 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/4d61090df7f242afac304de805fdacf6 2024-12-09T08:14:21,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741901_1077 (size=12301) 2024-12-09T08:14:21,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/211b217f282048609bb9418b0abb8ba6 2024-12-09T08:14:21,983 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into 4d61090df7f242afac304de805fdacf6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:21,983 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:21,983 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=13, startTime=1733732061814; duration=0sec 2024-12-09T08:14:21,983 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:21,983 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:21,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/146889fb8a624befb7dd01ababde6de8 is 50, key is test_row_0/C:col10/1733732061872/Put/seqid=0 2024-12-09T08:14:22,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741902_1078 (size=12301) 2024-12-09T08:14:22,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/146889fb8a624befb7dd01ababde6de8 2024-12-09T08:14:22,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/e622f3285aca4ce792c90431a1154955 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e622f3285aca4ce792c90431a1154955 2024-12-09T08:14:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T08:14:22,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e622f3285aca4ce792c90431a1154955, entries=200, sequenceid=353, filesize=14.4 K 2024-12-09T08:14:22,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/211b217f282048609bb9418b0abb8ba6 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/211b217f282048609bb9418b0abb8ba6 2024-12-09T08:14:22,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/211b217f282048609bb9418b0abb8ba6, entries=150, sequenceid=353, filesize=12.0 K 2024-12-09T08:14:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/146889fb8a624befb7dd01ababde6de8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/146889fb8a624befb7dd01ababde6de8 2024-12-09T08:14:22,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/146889fb8a624befb7dd01ababde6de8, entries=150, sequenceid=353, filesize=12.0 K 2024-12-09T08:14:22,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 589b933f5d38176578d90952a67fb0a1 in 182ms, sequenceid=353, compaction requested=false 2024-12-09T08:14:22,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:22,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T08:14:22,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:22,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:22,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:22,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:22,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:22,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:22,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/e411022198624a739af6c76e381be9fd is 50, key is test_row_0/A:col10/1733732061927/Put/seqid=0 2024-12-09T08:14:22,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741903_1079 (size=14741) 2024-12-09T08:14:22,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T08:14:22,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:22,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:22,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:22,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:22,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:22,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/e411022198624a739af6c76e381be9fd 2024-12-09T08:14:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:22,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/3b5c66aa94f242c586dbffe24186281f is 50, key is test_row_0/B:col10/1733732061927/Put/seqid=0 2024-12-09T08:14:22,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741904_1080 (size=12301) 2024-12-09T08:14:22,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/3b5c66aa94f242c586dbffe24186281f 2024-12-09T08:14:22,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/5e0f0b8c711648988e24d00729585b21 is 50, key is test_row_0/C:col10/1733732061927/Put/seqid=0 2024-12-09T08:14:22,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741905_1081 (size=12301) 2024-12-09T08:14:22,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/5e0f0b8c711648988e24d00729585b21 2024-12-09T08:14:22,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/e411022198624a739af6c76e381be9fd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e411022198624a739af6c76e381be9fd 2024-12-09T08:14:22,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e411022198624a739af6c76e381be9fd, entries=200, sequenceid=369, filesize=14.4 K 2024-12-09T08:14:22,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732122161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:22,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/3b5c66aa94f242c586dbffe24186281f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/3b5c66aa94f242c586dbffe24186281f 2024-12-09T08:14:22,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/3b5c66aa94f242c586dbffe24186281f, entries=150, sequenceid=369, filesize=12.0 K 2024-12-09T08:14:22,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/5e0f0b8c711648988e24d00729585b21 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/5e0f0b8c711648988e24d00729585b21 2024-12-09T08:14:22,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/5e0f0b8c711648988e24d00729585b21, entries=150, sequenceid=369, filesize=12.0 K 2024-12-09T08:14:22,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 589b933f5d38176578d90952a67fb0a1 in 118ms, sequenceid=369, compaction requested=true 2024-12-09T08:14:22,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:22,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:22,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:22,182 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-12-09T08:14:22,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:22,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:22,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:22,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T08:14:22,183 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T08:14:22,183 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T08:14:22,183 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. because compaction request was cancelled 2024-12-09T08:14:22,183 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:22,183 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:22,184 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:22,184 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:22,185 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:22,185 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/704756a15fce49b09dedd2a95b659730, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/211b217f282048609bb9418b0abb8ba6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/3b5c66aa94f242c586dbffe24186281f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=36.7 K 2024-12-09T08:14:22,185 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 704756a15fce49b09dedd2a95b659730, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733732060509 2024-12-09T08:14:22,186 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 211b217f282048609bb9418b0abb8ba6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733732061751 2024-12-09T08:14:22,186 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b5c66aa94f242c586dbffe24186281f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733732061927 2024-12-09T08:14:22,196 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:22,197 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/c676ca8668b3432cadb28041e33e1c28 is 50, key is test_row_0/B:col10/1733732061927/Put/seqid=0 2024-12-09T08:14:22,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741906_1082 (size=13085) 2024-12-09T08:14:22,215 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/c676ca8668b3432cadb28041e33e1c28 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/c676ca8668b3432cadb28041e33e1c28 2024-12-09T08:14:22,222 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into c676ca8668b3432cadb28041e33e1c28(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:22,222 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:22,222 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=13, startTime=1733732062182; duration=0sec 2024-12-09T08:14:22,222 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:22,222 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:22,222 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:22,224 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:22,224 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:22,224 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:22,224 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/4d61090df7f242afac304de805fdacf6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/146889fb8a624befb7dd01ababde6de8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/5e0f0b8c711648988e24d00729585b21] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=36.7 K 2024-12-09T08:14:22,225 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d61090df7f242afac304de805fdacf6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733732060509 2024-12-09T08:14:22,226 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 146889fb8a624befb7dd01ababde6de8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733732061751 2024-12-09T08:14:22,226 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e0f0b8c711648988e24d00729585b21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733732061927 2024-12-09T08:14:22,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T08:14:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:22,239 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T08:14:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:22,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:22,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:22,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:22,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:22,241 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T08:14:22,243 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/8bccaaa842ed4ca087eb9c161add8612 is 50, key is test_row_0/C:col10/1733732061927/Put/seqid=0 2024-12-09T08:14:22,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/2a899e3c41ea425b8a869f517785f1bc is 50, key is test_row_0/A:col10/1733732062136/Put/seqid=0 2024-12-09T08:14:22,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741907_1083 (size=13085) 2024-12-09T08:14:22,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741908_1084 (size=12301) 2024-12-09T08:14:22,266 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/2a899e3c41ea425b8a869f517785f1bc 2024-12-09T08:14:22,272 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/99ba7731daec4e8780fcd68080d52cac as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/99ba7731daec4e8780fcd68080d52cac 2024-12-09T08:14:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:22,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:22,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/2e0f7d7c1465419199e30a500e21a14c is 50, key is test_row_0/B:col10/1733732062136/Put/seqid=0 2024-12-09T08:14:22,280 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/A of 589b933f5d38176578d90952a67fb0a1 into 99ba7731daec4e8780fcd68080d52cac(size=12.7 K), total size for store is 41.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:22,280 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:22,280 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/A, priority=13, startTime=1733732061811; duration=0sec 2024-12-09T08:14:22,281 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:22,281 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:22,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741909_1085 (size=12301) 2024-12-09T08:14:22,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732122327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:22,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732122441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T08:14:22,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732122651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:22,671 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/8bccaaa842ed4ca087eb9c161add8612 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8bccaaa842ed4ca087eb9c161add8612 2024-12-09T08:14:22,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732122671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:22,673 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:22,678 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into 8bccaaa842ed4ca087eb9c161add8612(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:22,678 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:22,678 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=13, startTime=1733732062182; duration=0sec 2024-12-09T08:14:22,678 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:22,678 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:22,682 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/2e0f7d7c1465419199e30a500e21a14c 2024-12-09T08:14:22,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732122681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:22,685 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:22,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:22,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:22,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/a4511dd31ce94132a5ac59cf1c484107 is 50, key is test_row_0/C:col10/1733732062136/Put/seqid=0 2024-12-09T08:14:22,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741910_1086 (size=12301) 2024-12-09T08:14:22,699 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/a4511dd31ce94132a5ac59cf1c484107 2024-12-09T08:14:22,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/2a899e3c41ea425b8a869f517785f1bc as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/2a899e3c41ea425b8a869f517785f1bc 2024-12-09T08:14:22,713 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/2a899e3c41ea425b8a869f517785f1bc, entries=150, sequenceid=388, filesize=12.0 K 2024-12-09T08:14:22,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/2e0f7d7c1465419199e30a500e21a14c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/2e0f7d7c1465419199e30a500e21a14c 2024-12-09T08:14:22,720 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/2e0f7d7c1465419199e30a500e21a14c, entries=150, sequenceid=388, filesize=12.0 K 2024-12-09T08:14:22,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/a4511dd31ce94132a5ac59cf1c484107 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a4511dd31ce94132a5ac59cf1c484107 2024-12-09T08:14:22,727 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a4511dd31ce94132a5ac59cf1c484107, entries=150, sequenceid=388, filesize=12.0 K 2024-12-09T08:14:22,729 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 589b933f5d38176578d90952a67fb0a1 in 489ms, sequenceid=388, compaction requested=true 2024-12-09T08:14:22,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:22,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:22,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-09T08:14:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-09T08:14:22,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-09T08:14:22,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 803 msec 2024-12-09T08:14:22,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 811 msec 2024-12-09T08:14:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:22,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T08:14:22,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:22,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:22,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:22,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:22,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:22,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:22,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/ab44b2262f8848aa989e8fb2cc4e0b5d is 50, key is test_row_0/A:col10/1733732062301/Put/seqid=0 2024-12-09T08:14:22,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741911_1087 (size=14741) 2024-12-09T08:14:23,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732123030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T08:14:23,061 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:23,062 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-09T08:14:23,065 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T08:14:23,066 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:23,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:23,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732123141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:23,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T08:14:23,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732123353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:23,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:23,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/ab44b2262f8848aa989e8fb2cc4e0b5d 2024-12-09T08:14:23,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T08:14:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/0e624d3d61c94f56a2742addafea417f is 50, key is test_row_0/B:col10/1733732062301/Put/seqid=0 2024-12-09T08:14:23,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741912_1088 (size=12301) 2024-12-09T08:14:23,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/0e624d3d61c94f56a2742addafea417f 2024-12-09T08:14:23,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/5230528ec3c2426bb4b42b4ed36da1ec is 50, key is test_row_0/C:col10/1733732062301/Put/seqid=0 2024-12-09T08:14:23,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741913_1089 (size=12301) 2024-12-09T08:14:23,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/5230528ec3c2426bb4b42b4ed36da1ec 2024-12-09T08:14:23,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/ab44b2262f8848aa989e8fb2cc4e0b5d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ab44b2262f8848aa989e8fb2cc4e0b5d 2024-12-09T08:14:23,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ab44b2262f8848aa989e8fb2cc4e0b5d, entries=200, sequenceid=405, filesize=14.4 K 2024-12-09T08:14:23,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/0e624d3d61c94f56a2742addafea417f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/0e624d3d61c94f56a2742addafea417f 2024-12-09T08:14:23,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/0e624d3d61c94f56a2742addafea417f, entries=150, sequenceid=405, filesize=12.0 K 2024-12-09T08:14:23,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/5230528ec3c2426bb4b42b4ed36da1ec as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/5230528ec3c2426bb4b42b4ed36da1ec 2024-12-09T08:14:23,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/5230528ec3c2426bb4b42b4ed36da1ec, entries=150, sequenceid=405, filesize=12.0 K 2024-12-09T08:14:23,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 589b933f5d38176578d90952a67fb0a1 in 608ms, sequenceid=405, compaction requested=true 2024-12-09T08:14:23,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:23,573 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T08:14:23,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:23,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:23,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:23,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:23,574 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:23,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:23,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:23,578 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:23,578 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:23,578 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,579 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/c676ca8668b3432cadb28041e33e1c28, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/2e0f7d7c1465419199e30a500e21a14c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/0e624d3d61c94f56a2742addafea417f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=36.8 K 2024-12-09T08:14:23,579 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 69507 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T08:14:23,579 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/A is initiating minor compaction (all files) 2024-12-09T08:14:23,579 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/A in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,579 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/99ba7731daec4e8780fcd68080d52cac, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e622f3285aca4ce792c90431a1154955, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e411022198624a739af6c76e381be9fd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/2a899e3c41ea425b8a869f517785f1bc, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ab44b2262f8848aa989e8fb2cc4e0b5d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=67.9 K 2024-12-09T08:14:23,580 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting c676ca8668b3432cadb28041e33e1c28, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733732061927 2024-12-09T08:14:23,580 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99ba7731daec4e8780fcd68080d52cac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733732060509 2024-12-09T08:14:23,580 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e0f7d7c1465419199e30a500e21a14c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733732062136 2024-12-09T08:14:23,581 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e624d3d61c94f56a2742addafea417f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733732062301 2024-12-09T08:14:23,581 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e622f3285aca4ce792c90431a1154955, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733732061706 2024-12-09T08:14:23,582 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e411022198624a739af6c76e381be9fd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733732061907 2024-12-09T08:14:23,583 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a899e3c41ea425b8a869f517785f1bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733732062136 2024-12-09T08:14:23,583 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab44b2262f8848aa989e8fb2cc4e0b5d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733732062301 2024-12-09T08:14:23,596 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#77 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:23,597 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/a98ed9a55b614c7993df3772ded009fa is 50, key is test_row_0/B:col10/1733732062301/Put/seqid=0 2024-12-09T08:14:23,608 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#A#compaction#78 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:23,609 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/a81fa37bdb5048a3b64dca6678acb110 is 50, key is test_row_0/A:col10/1733732062301/Put/seqid=0 2024-12-09T08:14:23,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741914_1090 (size=13187) 2024-12-09T08:14:23,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741915_1091 (size=13153) 2024-12-09T08:14:23,644 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/a98ed9a55b614c7993df3772ded009fa as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/a98ed9a55b614c7993df3772ded009fa 2024-12-09T08:14:23,653 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/a81fa37bdb5048a3b64dca6678acb110 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a81fa37bdb5048a3b64dca6678acb110 2024-12-09T08:14:23,660 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into a98ed9a55b614c7993df3772ded009fa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:23,660 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:23,660 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=13, startTime=1733732063574; duration=0sec 2024-12-09T08:14:23,662 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:23,662 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:23,662 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:23,664 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:23,664 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:23,664 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,664 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8bccaaa842ed4ca087eb9c161add8612, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a4511dd31ce94132a5ac59cf1c484107, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/5230528ec3c2426bb4b42b4ed36da1ec] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=36.8 K 2024-12-09T08:14:23,665 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bccaaa842ed4ca087eb9c161add8612, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733732061927 2024-12-09T08:14:23,666 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a4511dd31ce94132a5ac59cf1c484107, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733732062136 2024-12-09T08:14:23,667 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5230528ec3c2426bb4b42b4ed36da1ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733732062301 2024-12-09T08:14:23,667 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/A of 589b933f5d38176578d90952a67fb0a1 into a81fa37bdb5048a3b64dca6678acb110(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:23,667 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:23,668 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/A, priority=11, startTime=1733732063573; duration=0sec 2024-12-09T08:14:23,668 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:23,668 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:23,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T08:14:23,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:23,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:23,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:23,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:23,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:23,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:23,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/81aea940785040e0bfdd98aa151e2a98 is 50, key is test_row_0/A:col10/1733732063672/Put/seqid=0 2024-12-09T08:14:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T08:14:23,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:23,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:23,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,700 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:23,701 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/44a8dcd5f6cf437c97551ba885c233af is 50, key is test_row_0/C:col10/1733732062301/Put/seqid=0 2024-12-09T08:14:23,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741917_1093 (size=13187) 2024-12-09T08:14:23,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741916_1092 (size=14741) 2024-12-09T08:14:23,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/81aea940785040e0bfdd98aa151e2a98 2024-12-09T08:14:23,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732123732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:23,740 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/44a8dcd5f6cf437c97551ba885c233af as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/44a8dcd5f6cf437c97551ba885c233af 2024-12-09T08:14:23,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/76d572709eea4dcbbe284ecd868445c4 is 50, key is test_row_0/B:col10/1733732063672/Put/seqid=0 2024-12-09T08:14:23,750 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into 44a8dcd5f6cf437c97551ba885c233af(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:23,750 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:23,750 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=13, startTime=1733732063574; duration=0sec 2024-12-09T08:14:23,750 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:23,751 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:23,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741918_1094 (size=12301) 2024-12-09T08:14:23,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/76d572709eea4dcbbe284ecd868445c4 2024-12-09T08:14:23,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/562ec94dc89447678e6dabfc60ebef42 is 50, key is test_row_0/C:col10/1733732063672/Put/seqid=0 2024-12-09T08:14:23,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741919_1095 (size=12301) 2024-12-09T08:14:23,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/562ec94dc89447678e6dabfc60ebef42 2024-12-09T08:14:23,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732123841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:23,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:23,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:23,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:23,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:23,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/81aea940785040e0bfdd98aa151e2a98 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/81aea940785040e0bfdd98aa151e2a98 2024-12-09T08:14:23,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/81aea940785040e0bfdd98aa151e2a98, entries=200, sequenceid=425, filesize=14.4 K 2024-12-09T08:14:23,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/76d572709eea4dcbbe284ecd868445c4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/76d572709eea4dcbbe284ecd868445c4 2024-12-09T08:14:23,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/76d572709eea4dcbbe284ecd868445c4, entries=150, sequenceid=425, filesize=12.0 K 2024-12-09T08:14:23,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/562ec94dc89447678e6dabfc60ebef42 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/562ec94dc89447678e6dabfc60ebef42 2024-12-09T08:14:23,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/562ec94dc89447678e6dabfc60ebef42, entries=150, sequenceid=425, filesize=12.0 K 2024-12-09T08:14:23,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 589b933f5d38176578d90952a67fb0a1 in 202ms, sequenceid=425, compaction requested=false 2024-12-09T08:14:23,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:23,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T08:14:23,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:23,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:23,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:23,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:23,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:23,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:23,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/146e2e264fde41a1b7fcfa67c383e4f8 is 50, key is test_row_0/A:col10/1733732063710/Put/seqid=0 2024-12-09T08:14:23,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741920_1096 (size=12301) 2024-12-09T08:14:23,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732123955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:23,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732123955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:24,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:24,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:24,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732124051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:24,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732124061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:24,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:24,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732124071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:24,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:24,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:24,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:24,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T08:14:24,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732124271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:24,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732124280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:24,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:24,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:24,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/146e2e264fde41a1b7fcfa67c383e4f8 2024-12-09T08:14:24,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/4e69b1f9938e4dc3a4dc06cd43240cbc is 50, key is test_row_0/B:col10/1733732063710/Put/seqid=0 2024-12-09T08:14:24,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741921_1097 (size=12301) 2024-12-09T08:14:24,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732124361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:24,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732124581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:24,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732124591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:24,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:24,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:24,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732124681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:24,685 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:24,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732124691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:24,695 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:24,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/4e69b1f9938e4dc3a4dc06cd43240cbc 2024-12-09T08:14:24,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/a660e2d0acdb41029711b4ccf7178c04 is 50, key is test_row_0/C:col10/1733732063710/Put/seqid=0 2024-12-09T08:14:24,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741922_1098 (size=12301) 2024-12-09T08:14:24,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732124871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:24,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:24,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:25,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:25,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:25,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732125090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:25,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732125111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:25,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:25,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:25,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:25,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/a660e2d0acdb41029711b4ccf7178c04 2024-12-09T08:14:25,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/146e2e264fde41a1b7fcfa67c383e4f8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/146e2e264fde41a1b7fcfa67c383e4f8 2024-12-09T08:14:25,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/146e2e264fde41a1b7fcfa67c383e4f8, entries=150, sequenceid=441, filesize=12.0 K 2024-12-09T08:14:25,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/4e69b1f9938e4dc3a4dc06cd43240cbc as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/4e69b1f9938e4dc3a4dc06cd43240cbc 2024-12-09T08:14:25,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/4e69b1f9938e4dc3a4dc06cd43240cbc, entries=150, sequenceid=441, filesize=12.0 K 2024-12-09T08:14:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/a660e2d0acdb41029711b4ccf7178c04 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a660e2d0acdb41029711b4ccf7178c04 2024-12-09T08:14:25,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a660e2d0acdb41029711b4ccf7178c04, entries=150, sequenceid=441, filesize=12.0 K 2024-12-09T08:14:25,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 589b933f5d38176578d90952a67fb0a1 in 1279ms, sequenceid=441, compaction requested=true 2024-12-09T08:14:25,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:25,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:25,174 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:25,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:25,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:25,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:25,174 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:25,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:25,175 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:25,175 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:25,175 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/A is initiating minor compaction (all files) 2024-12-09T08:14:25,175 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:25,175 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/A in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:25,175 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:25,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:25,175 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a81fa37bdb5048a3b64dca6678acb110, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/81aea940785040e0bfdd98aa151e2a98, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/146e2e264fde41a1b7fcfa67c383e4f8] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=39.3 K 2024-12-09T08:14:25,175 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/a98ed9a55b614c7993df3772ded009fa, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/76d572709eea4dcbbe284ecd868445c4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/4e69b1f9938e4dc3a4dc06cd43240cbc] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=36.9 K 2024-12-09T08:14:25,176 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a81fa37bdb5048a3b64dca6678acb110, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733732062301 2024-12-09T08:14:25,176 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a98ed9a55b614c7993df3772ded009fa, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733732062301 2024-12-09T08:14:25,176 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81aea940785040e0bfdd98aa151e2a98, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733732063018 2024-12-09T08:14:25,176 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 76d572709eea4dcbbe284ecd868445c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733732063023 2024-12-09T08:14:25,177 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 146e2e264fde41a1b7fcfa67c383e4f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733732063710 2024-12-09T08:14:25,177 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e69b1f9938e4dc3a4dc06cd43240cbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733732063710 2024-12-09T08:14:25,187 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:25,188 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/43fc1eb7024440108ac214040ac3e645 is 50, key is test_row_0/B:col10/1733732063710/Put/seqid=0 2024-12-09T08:14:25,192 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:25,193 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/a3ee91209661476bb5b5d99c9f61048a is 50, key is test_row_0/A:col10/1733732063710/Put/seqid=0 2024-12-09T08:14:25,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741924_1100 (size=13255) 2024-12-09T08:14:25,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741923_1099 (size=13289) 2024-12-09T08:14:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T08:14:25,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T08:14:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:25,229 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T08:14:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/bbd0c847d47046d8bc3635114ac3a043 is 50, key is test_row_0/A:col10/1733732063953/Put/seqid=0 2024-12-09T08:14:25,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741925_1101 (size=12301) 2024-12-09T08:14:25,608 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/a3ee91209661476bb5b5d99c9f61048a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a3ee91209661476bb5b5d99c9f61048a 2024-12-09T08:14:25,612 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/43fc1eb7024440108ac214040ac3e645 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/43fc1eb7024440108ac214040ac3e645 2024-12-09T08:14:25,615 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/A of 589b933f5d38176578d90952a67fb0a1 into a3ee91209661476bb5b5d99c9f61048a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:25,615 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:25,615 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/A, priority=13, startTime=1733732065173; duration=0sec 2024-12-09T08:14:25,615 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:25,615 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:25,615 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:25,616 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:25,616 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:25,616 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:25,617 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/44a8dcd5f6cf437c97551ba885c233af, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/562ec94dc89447678e6dabfc60ebef42, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a660e2d0acdb41029711b4ccf7178c04] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=36.9 K 2024-12-09T08:14:25,617 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44a8dcd5f6cf437c97551ba885c233af, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733732062301 2024-12-09T08:14:25,617 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 562ec94dc89447678e6dabfc60ebef42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733732063023 2024-12-09T08:14:25,618 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a660e2d0acdb41029711b4ccf7178c04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733732063710 2024-12-09T08:14:25,618 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into 43fc1eb7024440108ac214040ac3e645(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:25,618 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:25,618 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=13, startTime=1733732065174; duration=0sec 2024-12-09T08:14:25,618 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:25,618 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:25,627 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:25,628 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/1f1853bbf3bd483b88c9e6a12f066d73 is 50, key is test_row_0/C:col10/1733732063710/Put/seqid=0 2024-12-09T08:14:25,639 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/bbd0c847d47046d8bc3635114ac3a043 2024-12-09T08:14:25,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741926_1102 (size=13289) 2024-12-09T08:14:25,663 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/1f1853bbf3bd483b88c9e6a12f066d73 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/1f1853bbf3bd483b88c9e6a12f066d73 2024-12-09T08:14:25,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/09535b5d328b406791c2c7283247a9ed is 50, key is test_row_0/B:col10/1733732063953/Put/seqid=0 2024-12-09T08:14:25,670 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into 1f1853bbf3bd483b88c9e6a12f066d73(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:25,670 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:25,670 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=13, startTime=1733732065174; duration=0sec 2024-12-09T08:14:25,670 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:25,670 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:25,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741927_1103 (size=12301) 2024-12-09T08:14:25,679 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/09535b5d328b406791c2c7283247a9ed 2024-12-09T08:14:25,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/7bfa3c40f3cf4d078e69e4f1dfe7d6f0 is 50, key is test_row_0/C:col10/1733732063953/Put/seqid=0 2024-12-09T08:14:25,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741928_1104 (size=12301) 2024-12-09T08:14:25,694 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/7bfa3c40f3cf4d078e69e4f1dfe7d6f0 2024-12-09T08:14:25,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/bbd0c847d47046d8bc3635114ac3a043 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bbd0c847d47046d8bc3635114ac3a043 2024-12-09T08:14:25,706 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bbd0c847d47046d8bc3635114ac3a043, entries=150, sequenceid=459, filesize=12.0 K 2024-12-09T08:14:25,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/09535b5d328b406791c2c7283247a9ed as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/09535b5d328b406791c2c7283247a9ed 2024-12-09T08:14:25,714 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/09535b5d328b406791c2c7283247a9ed, entries=150, sequenceid=459, filesize=12.0 K 2024-12-09T08:14:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/7bfa3c40f3cf4d078e69e4f1dfe7d6f0 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/7bfa3c40f3cf4d078e69e4f1dfe7d6f0 2024-12-09T08:14:25,722 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/7bfa3c40f3cf4d078e69e4f1dfe7d6f0, entries=150, sequenceid=459, filesize=12.0 K 2024-12-09T08:14:25,723 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 589b933f5d38176578d90952a67fb0a1 in 494ms, sequenceid=459, compaction requested=false 2024-12-09T08:14:25,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:25,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:25,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-09T08:14:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-09T08:14:25,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-09T08:14:25,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6590 sec 2024-12-09T08:14:25,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.6640 sec 2024-12-09T08:14:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:25,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:14:25,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:25,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:25,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:25,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:25,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:25,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:25,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/7fae2f6ecc9d413a8c64655c58a719d4 is 50, key is test_row_0/A:col10/1733732065896/Put/seqid=0 2024-12-09T08:14:25,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741929_1105 (size=14737) 2024-12-09T08:14:25,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732065959, max timestamp=1733732065974Num uniques 859; 2024-12-09T08:14:25,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732065959, max timestamp=1733732065974Num uniques 859; 2024-12-09T08:14:25,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732065959, max timestamp=1733732065974Num uniques 859; 2024-12-09T08:14:25,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:14:25,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:14:25,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:14:26,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732126033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732126101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:26,102 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:26,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732126131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:26,133 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:26,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732126140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:26,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/7fae2f6ecc9d413a8c64655c58a719d4 2024-12-09T08:14:26,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/1c84a6d4586c4ff1ae06dce35b31a77a is 50, key is test_row_0/B:col10/1733732065896/Put/seqid=0 2024-12-09T08:14:26,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732126351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:26,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741930_1106 (size=9857) 2024-12-09T08:14:26,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/1c84a6d4586c4ff1ae06dce35b31a77a 2024-12-09T08:14:26,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/8ad04f102da04492a4222a32609c4c10 is 50, key is test_row_0/C:col10/1733732065896/Put/seqid=0 2024-12-09T08:14:26,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741931_1107 (size=9857) 2024-12-09T08:14:26,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/8ad04f102da04492a4222a32609c4c10 2024-12-09T08:14:26,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/7fae2f6ecc9d413a8c64655c58a719d4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7fae2f6ecc9d413a8c64655c58a719d4 2024-12-09T08:14:26,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7fae2f6ecc9d413a8c64655c58a719d4, entries=200, sequenceid=472, filesize=14.4 K 2024-12-09T08:14:26,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/1c84a6d4586c4ff1ae06dce35b31a77a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1c84a6d4586c4ff1ae06dce35b31a77a 2024-12-09T08:14:26,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1c84a6d4586c4ff1ae06dce35b31a77a, entries=100, sequenceid=472, filesize=9.6 K 2024-12-09T08:14:26,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/8ad04f102da04492a4222a32609c4c10 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ad04f102da04492a4222a32609c4c10 2024-12-09T08:14:26,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ad04f102da04492a4222a32609c4c10, entries=100, sequenceid=472, filesize=9.6 K 2024-12-09T08:14:26,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 589b933f5d38176578d90952a67fb0a1 in 503ms, sequenceid=472, compaction requested=true 2024-12-09T08:14:26,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:26,410 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:26,410 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:26,411 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40293 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:26,411 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/A is initiating minor compaction (all files) 2024-12-09T08:14:26,412 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/A in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:26,412 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a3ee91209661476bb5b5d99c9f61048a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bbd0c847d47046d8bc3635114ac3a043, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7fae2f6ecc9d413a8c64655c58a719d4] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=39.3 K 2024-12-09T08:14:26,412 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3ee91209661476bb5b5d99c9f61048a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733732063710 2024-12-09T08:14:26,413 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbd0c847d47046d8bc3635114ac3a043, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733732063942 2024-12-09T08:14:26,413 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fae2f6ecc9d413a8c64655c58a719d4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733732065896 2024-12-09T08:14:26,414 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:26,414 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:26,414 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:26,414 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/43fc1eb7024440108ac214040ac3e645, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/09535b5d328b406791c2c7283247a9ed, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1c84a6d4586c4ff1ae06dce35b31a77a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=34.6 K 2024-12-09T08:14:26,415 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 43fc1eb7024440108ac214040ac3e645, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733732063710 2024-12-09T08:14:26,416 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 09535b5d328b406791c2c7283247a9ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733732063942 2024-12-09T08:14:26,416 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c84a6d4586c4ff1ae06dce35b31a77a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733732065896 2024-12-09T08:14:26,425 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#A#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:26,425 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/d538f048a39946d8989782f2394a356f is 50, key is test_row_0/A:col10/1733732065896/Put/seqid=0 2024-12-09T08:14:26,426 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:26,426 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/78ceca4171204137b03638bbc99221d9 is 50, key is test_row_0/B:col10/1733732065896/Put/seqid=0 2024-12-09T08:14:26,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741932_1108 (size=13357) 2024-12-09T08:14:26,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741933_1109 (size=13391) 2024-12-09T08:14:26,437 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/d538f048a39946d8989782f2394a356f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/d538f048a39946d8989782f2394a356f 2024-12-09T08:14:26,446 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/78ceca4171204137b03638bbc99221d9 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/78ceca4171204137b03638bbc99221d9 2024-12-09T08:14:26,446 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/A of 589b933f5d38176578d90952a67fb0a1 into d538f048a39946d8989782f2394a356f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:26,446 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:26,446 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/A, priority=13, startTime=1733732066410; duration=0sec 2024-12-09T08:14:26,446 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:26,446 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:26,447 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:26,448 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:26,448 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:26,448 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:26,448 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/1f1853bbf3bd483b88c9e6a12f066d73, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/7bfa3c40f3cf4d078e69e4f1dfe7d6f0, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ad04f102da04492a4222a32609c4c10] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=34.6 K 2024-12-09T08:14:26,449 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f1853bbf3bd483b88c9e6a12f066d73, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733732063710 2024-12-09T08:14:26,449 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bfa3c40f3cf4d078e69e4f1dfe7d6f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733732063942 2024-12-09T08:14:26,450 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad04f102da04492a4222a32609c4c10, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733732065896 2024-12-09T08:14:26,457 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into 78ceca4171204137b03638bbc99221d9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:26,457 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:26,457 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=13, startTime=1733732066410; duration=0sec 2024-12-09T08:14:26,458 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:26,458 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:26,461 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:26,463 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/64273f2805fb47e98be1acd24a3490e3 is 50, key is test_row_0/C:col10/1733732065896/Put/seqid=0 2024-12-09T08:14:26,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741934_1110 (size=13391) 2024-12-09T08:14:26,481 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/64273f2805fb47e98be1acd24a3490e3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/64273f2805fb47e98be1acd24a3490e3 2024-12-09T08:14:26,487 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into 64273f2805fb47e98be1acd24a3490e3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:26,487 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:26,487 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=13, startTime=1733732066410; duration=0sec 2024-12-09T08:14:26,488 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:26,488 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:26,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-09T08:14:26,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:26,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:26,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:26,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:26,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:26,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:26,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/e4ba0fe7f6c04fe2a376aa62f0f3b290 is 50, key is test_row_0/A:col10/1733732066662/Put/seqid=0 2024-12-09T08:14:26,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741935_1111 (size=14741) 2024-12-09T08:14:26,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/e4ba0fe7f6c04fe2a376aa62f0f3b290 2024-12-09T08:14:26,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732126685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:26,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/9eee5ae651a644c2a3ad15f74fcaa890 is 50, key is test_row_0/B:col10/1733732066662/Put/seqid=0 2024-12-09T08:14:26,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741936_1112 (size=12301) 2024-12-09T08:14:26,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732126791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:26,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:26,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:27,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732127001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:27,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/9eee5ae651a644c2a3ad15f74fcaa890 2024-12-09T08:14:27,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/092daa5cb5d74f0380832096409614b9 is 50, key is test_row_0/C:col10/1733732066662/Put/seqid=0 2024-12-09T08:14:27,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741937_1113 (size=12301) 2024-12-09T08:14:27,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/092daa5cb5d74f0380832096409614b9 2024-12-09T08:14:27,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/e4ba0fe7f6c04fe2a376aa62f0f3b290 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e4ba0fe7f6c04fe2a376aa62f0f3b290 2024-12-09T08:14:27,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e4ba0fe7f6c04fe2a376aa62f0f3b290, entries=200, sequenceid=512, filesize=14.4 K 2024-12-09T08:14:27,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/9eee5ae651a644c2a3ad15f74fcaa890 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/9eee5ae651a644c2a3ad15f74fcaa890 2024-12-09T08:14:27,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/9eee5ae651a644c2a3ad15f74fcaa890, entries=150, sequenceid=512, filesize=12.0 K 2024-12-09T08:14:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/092daa5cb5d74f0380832096409614b9 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/092daa5cb5d74f0380832096409614b9 2024-12-09T08:14:27,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/092daa5cb5d74f0380832096409614b9, entries=150, sequenceid=512, filesize=12.0 K 2024-12-09T08:14:27,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 589b933f5d38176578d90952a67fb0a1 in 486ms, sequenceid=512, compaction requested=false 2024-12-09T08:14:27,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T08:14:27,231 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:27,232 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-09T08:14:27,234 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T08:14:27,235 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:27,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:27,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:14:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:27,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/549569865e3149fd87362c1fed798322 is 50, key is test_row_0/A:col10/1733732066683/Put/seqid=0 2024-12-09T08:14:27,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741938_1114 (size=12301) 2024-12-09T08:14:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T08:14:27,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732067355, max timestamp=1733732067383Num uniques 859; 2024-12-09T08:14:27,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:14:27,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732067355, max timestamp=1733732067383Num uniques 859; 2024-12-09T08:14:27,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:14:27,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732067355, max timestamp=1733732067383Num uniques 859; 2024-12-09T08:14:27,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:14:27,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T08:14:27,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:27,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:27,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:27,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:27,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:27,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732127445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:27,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T08:14:27,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:27,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:27,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:27,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:27,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T08:14:27,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732127551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:27,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:27,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:27,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:27,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T08:14:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:27,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:27,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/549569865e3149fd87362c1fed798322 2024-12-09T08:14:27,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/b8731343d2bd4ac1b7364570de3d3cf8 is 50, key is test_row_0/B:col10/1733732066683/Put/seqid=0 2024-12-09T08:14:27,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741939_1115 (size=12301) 2024-12-09T08:14:27,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/b8731343d2bd4ac1b7364570de3d3cf8 2024-12-09T08:14:27,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732127761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:27,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/32c4194b51ab421ab4b5941d52c3d627 is 50, key is test_row_0/C:col10/1733732066683/Put/seqid=0 2024-12-09T08:14:27,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741940_1116 (size=12301) 2024-12-09T08:14:27,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T08:14:27,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:27,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:27,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:27,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:27,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T08:14:28,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T08:14:28,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:28,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:28,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:28,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:28,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:28,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732128072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:28,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:28,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1733732128121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:28,126 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:28,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1733732128151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:28,155 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:28,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T08:14:28,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:28,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:28,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:28,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:28,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/32c4194b51ab421ab4b5941d52c3d627 2024-12-09T08:14:28,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/549569865e3149fd87362c1fed798322 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/549569865e3149fd87362c1fed798322 2024-12-09T08:14:28,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/549569865e3149fd87362c1fed798322, entries=150, sequenceid=522, filesize=12.0 K 2024-12-09T08:14:28,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/b8731343d2bd4ac1b7364570de3d3cf8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/b8731343d2bd4ac1b7364570de3d3cf8 2024-12-09T08:14:28,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/b8731343d2bd4ac1b7364570de3d3cf8, entries=150, sequenceid=522, filesize=12.0 K 2024-12-09T08:14:28,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/32c4194b51ab421ab4b5941d52c3d627 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/32c4194b51ab421ab4b5941d52c3d627 2024-12-09T08:14:28,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/32c4194b51ab421ab4b5941d52c3d627, entries=150, sequenceid=522, filesize=12.0 K 2024-12-09T08:14:28,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 589b933f5d38176578d90952a67fb0a1 in 895ms, sequenceid=522, compaction requested=true 2024-12-09T08:14:28,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:28,209 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:28,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:28,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:28,210 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:28,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:28,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:28,210 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:28,210 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/A is initiating minor compaction (all files) 2024-12-09T08:14:28,211 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/A in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:28,211 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/d538f048a39946d8989782f2394a356f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e4ba0fe7f6c04fe2a376aa62f0f3b290, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/549569865e3149fd87362c1fed798322] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=39.5 K 2024-12-09T08:14:28,211 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:28,211 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:28,211 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:28,211 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/78ceca4171204137b03638bbc99221d9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/9eee5ae651a644c2a3ad15f74fcaa890, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/b8731343d2bd4ac1b7364570de3d3cf8] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=37.1 K 2024-12-09T08:14:28,211 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d538f048a39946d8989782f2394a356f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733732063942 2024-12-09T08:14:28,212 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4ba0fe7f6c04fe2a376aa62f0f3b290, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733732066021 2024-12-09T08:14:28,212 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 78ceca4171204137b03638bbc99221d9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733732063942 2024-12-09T08:14:28,212 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 549569865e3149fd87362c1fed798322, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733732066664 2024-12-09T08:14:28,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:28,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:28,212 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eee5ae651a644c2a3ad15f74fcaa890, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733732066021 2024-12-09T08:14:28,213 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting b8731343d2bd4ac1b7364570de3d3cf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733732066664 2024-12-09T08:14:28,237 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#A#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:28,238 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/4fde9f6a39bb4dd5930942c771ed92b8 is 50, key is test_row_0/A:col10/1733732066683/Put/seqid=0 2024-12-09T08:14:28,240 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:28,240 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/2ec9819c044f49f587e9e7373ccb45ca is 50, key is test_row_0/B:col10/1733732066683/Put/seqid=0 2024-12-09T08:14:28,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741941_1117 (size=13459) 2024-12-09T08:14:28,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741942_1118 (size=13493) 2024-12-09T08:14:28,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T08:14:28,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:28,310 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T08:14:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/fa320db4850a48a38a4985543c137de5 is 50, key is test_row_0/A:col10/1733732067420/Put/seqid=0 2024-12-09T08:14:28,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741943_1119 (size=12301) 2024-12-09T08:14:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T08:14:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:28,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732128604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:28,659 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/4fde9f6a39bb4dd5930942c771ed92b8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/4fde9f6a39bb4dd5930942c771ed92b8 2024-12-09T08:14:28,659 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/2ec9819c044f49f587e9e7373ccb45ca as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/2ec9819c044f49f587e9e7373ccb45ca 2024-12-09T08:14:28,667 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/A of 589b933f5d38176578d90952a67fb0a1 into 4fde9f6a39bb4dd5930942c771ed92b8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:28,668 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:28,668 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/A, priority=13, startTime=1733732068209; duration=0sec 2024-12-09T08:14:28,668 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:28,668 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:28,668 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:28,669 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:28,669 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:28,669 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:28,671 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/64273f2805fb47e98be1acd24a3490e3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/092daa5cb5d74f0380832096409614b9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/32c4194b51ab421ab4b5941d52c3d627] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=37.1 K 2024-12-09T08:14:28,671 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64273f2805fb47e98be1acd24a3490e3, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733732063942 2024-12-09T08:14:28,672 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 092daa5cb5d74f0380832096409614b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733732066021 2024-12-09T08:14:28,672 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32c4194b51ab421ab4b5941d52c3d627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733732066664 2024-12-09T08:14:28,673 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into 2ec9819c044f49f587e9e7373ccb45ca(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:28,673 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:28,673 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=13, startTime=1733732068209; duration=0sec 2024-12-09T08:14:28,673 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:28,673 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:28,687 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:28,687 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/187a1e67cb664e0ea6607af0cd532f73 is 50, key is test_row_0/C:col10/1733732066683/Put/seqid=0 2024-12-09T08:14:28,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741944_1120 (size=13493) 2024-12-09T08:14:28,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48168 deadline: 1733732128691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:28,693 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:28,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732128710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:28,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:28,721 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/fa320db4850a48a38a4985543c137de5 2024-12-09T08:14:28,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48190 deadline: 1733732128721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:28,724 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 589b933f5d38176578d90952a67fb0a1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:14:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:28,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/c643dcc427d9451da15123cf9757c010 is 50, key is test_row_0/B:col10/1733732067420/Put/seqid=0 2024-12-09T08:14:28,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741945_1121 (size=12301) 2024-12-09T08:14:28,746 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/c643dcc427d9451da15123cf9757c010 2024-12-09T08:14:28,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/c91e373a7004467bb934e8b326b51806 is 50, key is test_row_0/C:col10/1733732067420/Put/seqid=0 2024-12-09T08:14:28,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741946_1122 (size=12301) 2024-12-09T08:14:28,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732128921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:29,099 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/187a1e67cb664e0ea6607af0cd532f73 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/187a1e67cb664e0ea6607af0cd532f73 2024-12-09T08:14:29,106 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into 187a1e67cb664e0ea6607af0cd532f73(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:29,106 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:29,106 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=13, startTime=1733732068212; duration=0sec 2024-12-09T08:14:29,106 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:29,106 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:29,175 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/c91e373a7004467bb934e8b326b51806 2024-12-09T08:14:29,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/fa320db4850a48a38a4985543c137de5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/fa320db4850a48a38a4985543c137de5 2024-12-09T08:14:29,188 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/fa320db4850a48a38a4985543c137de5, entries=150, sequenceid=558, filesize=12.0 K 2024-12-09T08:14:29,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/c643dcc427d9451da15123cf9757c010 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/c643dcc427d9451da15123cf9757c010 2024-12-09T08:14:29,197 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/c643dcc427d9451da15123cf9757c010, entries=150, sequenceid=558, filesize=12.0 K 2024-12-09T08:14:29,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/c91e373a7004467bb934e8b326b51806 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/c91e373a7004467bb934e8b326b51806 2024-12-09T08:14:29,209 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/c91e373a7004467bb934e8b326b51806, entries=150, sequenceid=558, filesize=12.0 K 2024-12-09T08:14:29,212 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 589b933f5d38176578d90952a67fb0a1 in 903ms, sequenceid=558, compaction requested=false 2024-12-09T08:14:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-09T08:14:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-09T08:14:29,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-09T08:14:29,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9800 sec 2024-12-09T08:14:29,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.9890 sec 2024-12-09T08:14:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:29,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:14:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:29,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/6b3eaf9f9c614256a78b6b689a9c6113 is 50, key is test_row_0/A:col10/1733732068602/Put/seqid=0 2024-12-09T08:14:29,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741947_1123 (size=14741) 2024-12-09T08:14:29,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732069280, max timestamp=1733732069302Num uniques 859; 2024-12-09T08:14:29,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732069280, max timestamp=1733732069302Num uniques 859; 2024-12-09T08:14:29,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732069280, max timestamp=1733732069302Num uniques 859; 2024-12-09T08:14:29,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:14:29,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:14:29,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:14:29,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732129373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T08:14:29,382 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:29,382 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-09T08:14:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T08:14:29,385 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:29,386 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:29,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:29,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732129481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T08:14:29,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T08:14:29,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:29,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:29,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:29,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:29,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:29,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/6b3eaf9f9c614256a78b6b689a9c6113 2024-12-09T08:14:29,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/690805c5b1954330bc232dd7fb762d2c is 50, key is test_row_0/B:col10/1733732068602/Put/seqid=0 2024-12-09T08:14:29,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741948_1124 (size=12301) 2024-12-09T08:14:29,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/690805c5b1954330bc232dd7fb762d2c 2024-12-09T08:14:29,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/465f1214ac1546cdb07f25d5130cf258 is 50, key is test_row_0/C:col10/1733732068602/Put/seqid=0 2024-12-09T08:14:29,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741949_1125 (size=12301) 2024-12-09T08:14:29,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T08:14:29,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:29,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732129691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:29,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:29,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:29,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:29,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T08:14:29,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T08:14:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:29,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:29,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T08:14:29,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:29,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:29,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:29,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:29,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:30,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:30,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732130001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T08:14:30,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/465f1214ac1546cdb07f25d5130cf258 2024-12-09T08:14:30,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/6b3eaf9f9c614256a78b6b689a9c6113 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/6b3eaf9f9c614256a78b6b689a9c6113 2024-12-09T08:14:30,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/6b3eaf9f9c614256a78b6b689a9c6113, entries=200, sequenceid=572, filesize=14.4 K 2024-12-09T08:14:30,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/690805c5b1954330bc232dd7fb762d2c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/690805c5b1954330bc232dd7fb762d2c 2024-12-09T08:14:30,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/690805c5b1954330bc232dd7fb762d2c, entries=150, sequenceid=572, filesize=12.0 K 2024-12-09T08:14:30,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/465f1214ac1546cdb07f25d5130cf258 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/465f1214ac1546cdb07f25d5130cf258 2024-12-09T08:14:30,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/465f1214ac1546cdb07f25d5130cf258, entries=150, sequenceid=572, filesize=12.0 K 2024-12-09T08:14:30,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 589b933f5d38176578d90952a67fb0a1 in 874ms, sequenceid=572, compaction requested=true 2024-12-09T08:14:30,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:30,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:30,108 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:30,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:30,108 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:30,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:30,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:30,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:30,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:30,109 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:30,109 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:30,109 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:30,109 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/A is initiating minor compaction (all files) 2024-12-09T08:14:30,109 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:30,109 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/A in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:30,109 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/4fde9f6a39bb4dd5930942c771ed92b8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/fa320db4850a48a38a4985543c137de5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/6b3eaf9f9c614256a78b6b689a9c6113] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=39.6 K 2024-12-09T08:14:30,109 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/2ec9819c044f49f587e9e7373ccb45ca, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/c643dcc427d9451da15123cf9757c010, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/690805c5b1954330bc232dd7fb762d2c] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=37.2 K 2024-12-09T08:14:30,109 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fde9f6a39bb4dd5930942c771ed92b8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733732066664 2024-12-09T08:14:30,110 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec9819c044f49f587e9e7373ccb45ca, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733732066664 2024-12-09T08:14:30,110 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa320db4850a48a38a4985543c137de5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1733732067418 2024-12-09T08:14:30,110 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting c643dcc427d9451da15123cf9757c010, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1733732067418 2024-12-09T08:14:30,110 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b3eaf9f9c614256a78b6b689a9c6113, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1733732068591 2024-12-09T08:14:30,110 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 690805c5b1954330bc232dd7fb762d2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1733732068591 2024-12-09T08:14:30,120 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#A#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:30,120 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/7aa19197fde44cf593297dab501c462d is 50, key is test_row_0/A:col10/1733732068602/Put/seqid=0 2024-12-09T08:14:30,131 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:30,132 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/ab08c076e880498a9f1a9b51367dd77d is 50, key is test_row_0/B:col10/1733732068602/Put/seqid=0 2024-12-09T08:14:30,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741950_1126 (size=13561) 2024-12-09T08:14:30,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741951_1127 (size=13595) 2024-12-09T08:14:30,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T08:14:30,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:30,151 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-09T08:14:30,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:30,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/4baa9144932647c5bd7f6a28caee0afd is 50, key is test_row_0/A:col10/1733732069342/Put/seqid=0 2024-12-09T08:14:30,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741952_1128 (size=12301) 2024-12-09T08:14:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:30,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. as already flushing 2024-12-09T08:14:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T08:14:30,556 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/ab08c076e880498a9f1a9b51367dd77d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/ab08c076e880498a9f1a9b51367dd77d 2024-12-09T08:14:30,558 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/7aa19197fde44cf593297dab501c462d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7aa19197fde44cf593297dab501c462d 2024-12-09T08:14:30,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732130553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:30,567 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/4baa9144932647c5bd7f6a28caee0afd 2024-12-09T08:14:30,578 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into ab08c076e880498a9f1a9b51367dd77d(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:30,578 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:30,578 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=13, startTime=1733732070108; duration=0sec 2024-12-09T08:14:30,578 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:30,578 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:30,578 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:30,582 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/A of 589b933f5d38176578d90952a67fb0a1 into 7aa19197fde44cf593297dab501c462d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:30,582 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:30,583 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/A, priority=13, startTime=1733732070107; duration=0sec 2024-12-09T08:14:30,583 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:30,583 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:30,583 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:30,583 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:30,583 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:30,583 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/187a1e67cb664e0ea6607af0cd532f73, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/c91e373a7004467bb934e8b326b51806, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/465f1214ac1546cdb07f25d5130cf258] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=37.2 K 2024-12-09T08:14:30,584 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 187a1e67cb664e0ea6607af0cd532f73, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733732066664 2024-12-09T08:14:30,584 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting c91e373a7004467bb934e8b326b51806, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1733732067418 2024-12-09T08:14:30,585 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 465f1214ac1546cdb07f25d5130cf258, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1733732068591 2024-12-09T08:14:30,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/0c058a6762ab49c18a7ae7ce7d53b9f7 is 50, key is test_row_0/B:col10/1733732069342/Put/seqid=0 2024-12-09T08:14:30,619 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:30,619 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/b7a542f582ce44d1868d5209d9cd8667 is 50, key is test_row_0/C:col10/1733732068602/Put/seqid=0 2024-12-09T08:14:30,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741954_1130 (size=13595) 2024-12-09T08:14:30,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732130661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:30,670 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/b7a542f582ce44d1868d5209d9cd8667 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/b7a542f582ce44d1868d5209d9cd8667 2024-12-09T08:14:30,680 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into b7a542f582ce44d1868d5209d9cd8667(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:30,681 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:30,681 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=13, startTime=1733732070108; duration=0sec 2024-12-09T08:14:30,681 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:30,681 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:30,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741953_1129 (size=12301) 2024-12-09T08:14:30,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:30,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1733732130871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:14:31,088 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/0c058a6762ab49c18a7ae7ce7d53b9f7 2024-12-09T08:14:31,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/0423dafa91b844aba4dfae8aafec02cf is 50, key is test_row_0/C:col10/1733732069342/Put/seqid=0 2024-12-09T08:14:31,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741955_1131 (size=12301) 2024-12-09T08:14:31,109 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/0423dafa91b844aba4dfae8aafec02cf 2024-12-09T08:14:31,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/4baa9144932647c5bd7f6a28caee0afd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/4baa9144932647c5bd7f6a28caee0afd 2024-12-09T08:14:31,126 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/4baa9144932647c5bd7f6a28caee0afd, entries=150, sequenceid=608, filesize=12.0 K 2024-12-09T08:14:31,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/0c058a6762ab49c18a7ae7ce7d53b9f7 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/0c058a6762ab49c18a7ae7ce7d53b9f7 2024-12-09T08:14:31,133 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-09T08:14:31,133 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:14:31,135 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-09T08:14:31,135 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:14:31,136 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-09T08:14:31,136 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:14:31,137 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/0c058a6762ab49c18a7ae7ce7d53b9f7, entries=150, sequenceid=608, filesize=12.0 K 2024-12-09T08:14:31,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/0423dafa91b844aba4dfae8aafec02cf as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/0423dafa91b844aba4dfae8aafec02cf 2024-12-09T08:14:31,139 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:31,139 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:31,139 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:31,139 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:31,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:14:31,140 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:31,140 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:14:31,141 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:31,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:14:31,141 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-09T08:14:31,141 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:14:31,141 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:31,141 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:31,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:14:31,146 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/0423dafa91b844aba4dfae8aafec02cf, entries=150, sequenceid=608, filesize=12.0 K 2024-12-09T08:14:31,147 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 589b933f5d38176578d90952a67fb0a1 in 996ms, sequenceid=608, compaction requested=false 2024-12-09T08:14:31,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:31,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:31,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-09T08:14:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-09T08:14:31,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-09T08:14:31,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7620 sec 2024-12-09T08:14:31,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.7680 sec 2024-12-09T08:14:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:31,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T08:14:31,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:31,183 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-09T08:14:31,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:31,183 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:14:31,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:31,183 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:31,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:31,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:31,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:31,183 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:31,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:14:31,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/2f801af8322749f0afc2cd2b8f5bfa1f is 50, key is test_row_0/A:col10/1733732070523/Put/seqid=0 2024-12-09T08:14:31,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741956_1132 (size=12301) 2024-12-09T08:14:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T08:14:31,531 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:31,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/2f801af8322749f0afc2cd2b8f5bfa1f 2024-12-09T08:14:31,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/25ea71ad080747ee843ffe7b1e89931f is 50, key is test_row_0/B:col10/1733732070523/Put/seqid=0 2024-12-09T08:14:31,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741957_1133 (size=12301) 2024-12-09T08:14:32,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/25ea71ad080747ee843ffe7b1e89931f 2024-12-09T08:14:32,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/47f057f09bd6454a8cac115f47b84ddb is 50, key is test_row_0/C:col10/1733732070523/Put/seqid=0 2024-12-09T08:14:32,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741958_1134 (size=12301) 2024-12-09T08:14:32,157 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-09T08:14:32,157 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:14:32,157 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:32,157 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:32,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:14:32,173 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-09T08:14:32,173 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:14:32,173 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:32,173 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:32,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:14:32,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/47f057f09bd6454a8cac115f47b84ddb 2024-12-09T08:14:32,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/2f801af8322749f0afc2cd2b8f5bfa1f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/2f801af8322749f0afc2cd2b8f5bfa1f 2024-12-09T08:14:32,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/2f801af8322749f0afc2cd2b8f5bfa1f, entries=150, sequenceid=623, filesize=12.0 K 2024-12-09T08:14:32,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/25ea71ad080747ee843ffe7b1e89931f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/25ea71ad080747ee843ffe7b1e89931f 2024-12-09T08:14:32,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/25ea71ad080747ee843ffe7b1e89931f, entries=150, sequenceid=623, filesize=12.0 K 2024-12-09T08:14:32,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/47f057f09bd6454a8cac115f47b84ddb as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/47f057f09bd6454a8cac115f47b84ddb 2024-12-09T08:14:32,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/47f057f09bd6454a8cac115f47b84ddb, entries=150, sequenceid=623, filesize=12.0 K 2024-12-09T08:14:32,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for 589b933f5d38176578d90952a67fb0a1 in 1273ms, sequenceid=623, compaction requested=true 2024-12-09T08:14:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:32,456 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:32,456 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 589b933f5d38176578d90952a67fb0a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:32,457 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:32,457 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:32,457 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/A is initiating minor compaction (all files) 2024-12-09T08:14:32,457 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/B is initiating minor compaction (all files) 2024-12-09T08:14:32,457 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/A in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:32,457 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/B in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:32,458 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/ab08c076e880498a9f1a9b51367dd77d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/0c058a6762ab49c18a7ae7ce7d53b9f7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/25ea71ad080747ee843ffe7b1e89931f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=37.3 K 2024-12-09T08:14:32,458 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7aa19197fde44cf593297dab501c462d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/4baa9144932647c5bd7f6a28caee0afd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/2f801af8322749f0afc2cd2b8f5bfa1f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=37.3 K 2024-12-09T08:14:32,458 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ab08c076e880498a9f1a9b51367dd77d, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1733732068591 2024-12-09T08:14:32,458 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aa19197fde44cf593297dab501c462d, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1733732068591 2024-12-09T08:14:32,458 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4baa9144932647c5bd7f6a28caee0afd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1733732069342 2024-12-09T08:14:32,458 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c058a6762ab49c18a7ae7ce7d53b9f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1733732069342 2024-12-09T08:14:32,459 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f801af8322749f0afc2cd2b8f5bfa1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1733732070523 2024-12-09T08:14:32,459 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ea71ad080747ee843ffe7b1e89931f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1733732070523 2024-12-09T08:14:32,468 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#A#compaction#122 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:32,469 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:32,469 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/b3ee14dc88bb4510aa5fba5af32fe560 is 50, key is test_row_0/A:col10/1733732070523/Put/seqid=0 2024-12-09T08:14:32,469 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/8fe1e278d1c844e5b179049a0640e09f is 50, key is test_row_0/B:col10/1733732070523/Put/seqid=0 2024-12-09T08:14:32,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741960_1136 (size=13663) 2024-12-09T08:14:32,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741959_1135 (size=13697) 2024-12-09T08:14:32,492 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/b3ee14dc88bb4510aa5fba5af32fe560 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/b3ee14dc88bb4510aa5fba5af32fe560 2024-12-09T08:14:32,497 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/A of 589b933f5d38176578d90952a67fb0a1 into b3ee14dc88bb4510aa5fba5af32fe560(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:32,497 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:32,498 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/A, priority=13, startTime=1733732072456; duration=0sec 2024-12-09T08:14:32,498 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:32,498 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:A 2024-12-09T08:14:32,498 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:32,499 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:32,499 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 589b933f5d38176578d90952a67fb0a1/C is initiating minor compaction (all files) 2024-12-09T08:14:32,499 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 589b933f5d38176578d90952a67fb0a1/C in TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:32,499 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/b7a542f582ce44d1868d5209d9cd8667, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/0423dafa91b844aba4dfae8aafec02cf, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/47f057f09bd6454a8cac115f47b84ddb] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp, totalSize=37.3 K 2024-12-09T08:14:32,499 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7a542f582ce44d1868d5209d9cd8667, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1733732068591 2024-12-09T08:14:32,500 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0423dafa91b844aba4dfae8aafec02cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1733732069342 2024-12-09T08:14:32,500 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47f057f09bd6454a8cac115f47b84ddb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1733732070523 2024-12-09T08:14:32,507 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 589b933f5d38176578d90952a67fb0a1#C#compaction#124 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:32,508 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/491b111fddf34df5b57b31b2460e774d is 50, key is test_row_0/C:col10/1733732070523/Put/seqid=0 2024-12-09T08:14:32,511 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T08:14:32,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741961_1137 (size=13697) 2024-12-09T08:14:32,895 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/8fe1e278d1c844e5b179049a0640e09f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/8fe1e278d1c844e5b179049a0640e09f 2024-12-09T08:14:32,900 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/B of 589b933f5d38176578d90952a67fb0a1 into 8fe1e278d1c844e5b179049a0640e09f(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:32,900 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:32,900 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/B, priority=13, startTime=1733732072456; duration=0sec 2024-12-09T08:14:32,900 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:32,901 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:B 2024-12-09T08:14:32,920 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/491b111fddf34df5b57b31b2460e774d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/491b111fddf34df5b57b31b2460e774d 2024-12-09T08:14:32,928 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 589b933f5d38176578d90952a67fb0a1/C of 589b933f5d38176578d90952a67fb0a1 into 491b111fddf34df5b57b31b2460e774d(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:32,928 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 589b933f5d38176578d90952a67fb0a1: 2024-12-09T08:14:32,928 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., storeName=589b933f5d38176578d90952a67fb0a1/C, priority=13, startTime=1733732072456; duration=0sec 2024-12-09T08:14:32,929 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:32,929 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 589b933f5d38176578d90952a67fb0a1:C 2024-12-09T08:14:38,744 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-09T08:14:38,744 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:14:38,744 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:38,745 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:38,745 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:14:38,824 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-09T08:14:38,824 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:14:38,824 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:38,824 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:38,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:14:38,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 260 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3499 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3587 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1604 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4810 rows 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1599 2024-12-09T08:14:38,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4793 rows 2024-12-09T08:14:38,825 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T08:14:38,826 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:14:38,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:38,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:38,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:14:38,831 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T08:14:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T08:14:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T08:14:38,839 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732078839"}]},"ts":"1733732078839"} 2024-12-09T08:14:38,841 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T08:14:38,841 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T08:14:38,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T08:14:38,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=589b933f5d38176578d90952a67fb0a1, UNASSIGN}] 2024-12-09T08:14:38,847 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=589b933f5d38176578d90952a67fb0a1, UNASSIGN 2024-12-09T08:14:38,848 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=589b933f5d38176578d90952a67fb0a1, regionState=CLOSING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:14:38,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=589b933f5d38176578d90952a67fb0a1, UNASSIGN because future has completed 2024-12-09T08:14:38,850 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:14:38,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:14:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T08:14:39,011 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:39,012 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:14:39,013 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 589b933f5d38176578d90952a67fb0a1, disabling compactions & flushes 2024-12-09T08:14:39,013 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:39,013 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:39,013 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. after waiting 0 ms 2024-12-09T08:14:39,013 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:39,013 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 589b933f5d38176578d90952a67fb0a1 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-09T08:14:39,014 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=A 2024-12-09T08:14:39,014 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:39,014 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=B 2024-12-09T08:14:39,014 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:39,014 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 589b933f5d38176578d90952a67fb0a1, store=C 2024-12-09T08:14:39,014 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:39,021 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/bdedc0e8ac274e4a9be2d3c79e72d126 is 50, key is test_row_0/A:col10/1733732072172/Put/seqid=0 2024-12-09T08:14:39,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741962_1138 (size=12301) 2024-12-09T08:14:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T08:14:39,427 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/bdedc0e8ac274e4a9be2d3c79e72d126 2024-12-09T08:14:39,440 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/7df9a35684f64cb1aa7ecd27addbbb99 is 50, key is test_row_0/B:col10/1733732072172/Put/seqid=0 2024-12-09T08:14:39,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741963_1139 (size=12301) 2024-12-09T08:14:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T08:14:39,846 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/7df9a35684f64cb1aa7ecd27addbbb99 2024-12-09T08:14:39,910 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/8ed9f7e43ad741a490e22c196a56d03b is 50, key is test_row_0/C:col10/1733732072172/Put/seqid=0 2024-12-09T08:14:39,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741964_1140 (size=12301) 2024-12-09T08:14:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T08:14:40,316 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/8ed9f7e43ad741a490e22c196a56d03b 2024-12-09T08:14:40,326 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/A/bdedc0e8ac274e4a9be2d3c79e72d126 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bdedc0e8ac274e4a9be2d3c79e72d126 2024-12-09T08:14:40,333 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bdedc0e8ac274e4a9be2d3c79e72d126, entries=150, sequenceid=633, filesize=12.0 K 2024-12-09T08:14:40,334 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/B/7df9a35684f64cb1aa7ecd27addbbb99 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/7df9a35684f64cb1aa7ecd27addbbb99 2024-12-09T08:14:40,339 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/7df9a35684f64cb1aa7ecd27addbbb99, entries=150, sequenceid=633, filesize=12.0 K 2024-12-09T08:14:40,340 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/.tmp/C/8ed9f7e43ad741a490e22c196a56d03b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ed9f7e43ad741a490e22c196a56d03b 2024-12-09T08:14:40,345 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ed9f7e43ad741a490e22c196a56d03b, entries=150, sequenceid=633, filesize=12.0 K 2024-12-09T08:14:40,345 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 589b933f5d38176578d90952a67fb0a1 in 1332ms, sequenceid=633, compaction requested=false 2024-12-09T08:14:40,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/56bd77eeb6094fc0a71f0db8f82791dd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/c0d0ba0fdff74453b4d6e39d773dac17, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7d7673ce2efe4e90a58b562d9215a65a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7f865960dbcb4569bb0a6275f5f0806f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bd02bc1c46ee42c3ad71f25c1e0b9ac8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7bbbee170da446bfb5c51851496b4f4d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ab0ce0c98241498daa0f07762ffbfb97, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/032cbfe696b043ffb013cd33bc302df3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/10c40b62a81240ae89ae7bc50b7bb57e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/0bd3d3072fc8477a83b1397cf69f29b5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/cd21012497ce468ea83cac93c271df5a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/613e3019dd3443978fc211c272761fce, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/d41ae45e4abd489e8f896a0173c4e577, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/6cf16d6554e54c1e9cd2db5ffe021aef, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/8710984f930142cfb3320927936ec8c7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/39849e29decd425896a2d192cd53fead, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/966dd50f0a034f2f833b48a432b4320b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a1adfa968a9146158956430f27f5b188, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ba0bbbb1a27247f7a4abfdb156fd11f0, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/99ba7731daec4e8780fcd68080d52cac, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/f920a8e9c7ad4a858a362a1650c1c501, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e622f3285aca4ce792c90431a1154955, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e411022198624a739af6c76e381be9fd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/2a899e3c41ea425b8a869f517785f1bc, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ab44b2262f8848aa989e8fb2cc4e0b5d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a81fa37bdb5048a3b64dca6678acb110, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/81aea940785040e0bfdd98aa151e2a98, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a3ee91209661476bb5b5d99c9f61048a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/146e2e264fde41a1b7fcfa67c383e4f8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bbd0c847d47046d8bc3635114ac3a043, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7fae2f6ecc9d413a8c64655c58a719d4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/d538f048a39946d8989782f2394a356f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e4ba0fe7f6c04fe2a376aa62f0f3b290, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/4fde9f6a39bb4dd5930942c771ed92b8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/549569865e3149fd87362c1fed798322, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/fa320db4850a48a38a4985543c137de5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/6b3eaf9f9c614256a78b6b689a9c6113, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7aa19197fde44cf593297dab501c462d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/4baa9144932647c5bd7f6a28caee0afd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/2f801af8322749f0afc2cd2b8f5bfa1f] to archive 2024-12-09T08:14:40,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:14:40,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/56bd77eeb6094fc0a71f0db8f82791dd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/56bd77eeb6094fc0a71f0db8f82791dd 2024-12-09T08:14:40,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/c0d0ba0fdff74453b4d6e39d773dac17 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/c0d0ba0fdff74453b4d6e39d773dac17 2024-12-09T08:14:40,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7d7673ce2efe4e90a58b562d9215a65a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7d7673ce2efe4e90a58b562d9215a65a 2024-12-09T08:14:40,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7f865960dbcb4569bb0a6275f5f0806f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7f865960dbcb4569bb0a6275f5f0806f 2024-12-09T08:14:40,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bd02bc1c46ee42c3ad71f25c1e0b9ac8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bd02bc1c46ee42c3ad71f25c1e0b9ac8 2024-12-09T08:14:40,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7bbbee170da446bfb5c51851496b4f4d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7bbbee170da446bfb5c51851496b4f4d 2024-12-09T08:14:40,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ab0ce0c98241498daa0f07762ffbfb97 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ab0ce0c98241498daa0f07762ffbfb97 2024-12-09T08:14:40,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/032cbfe696b043ffb013cd33bc302df3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/032cbfe696b043ffb013cd33bc302df3 2024-12-09T08:14:40,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/10c40b62a81240ae89ae7bc50b7bb57e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/10c40b62a81240ae89ae7bc50b7bb57e 2024-12-09T08:14:40,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/0bd3d3072fc8477a83b1397cf69f29b5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/0bd3d3072fc8477a83b1397cf69f29b5 2024-12-09T08:14:40,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/cd21012497ce468ea83cac93c271df5a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/cd21012497ce468ea83cac93c271df5a 2024-12-09T08:14:40,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/613e3019dd3443978fc211c272761fce to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/613e3019dd3443978fc211c272761fce 2024-12-09T08:14:40,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/d41ae45e4abd489e8f896a0173c4e577 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/d41ae45e4abd489e8f896a0173c4e577 2024-12-09T08:14:40,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/6cf16d6554e54c1e9cd2db5ffe021aef to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/6cf16d6554e54c1e9cd2db5ffe021aef 2024-12-09T08:14:40,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/8710984f930142cfb3320927936ec8c7 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/8710984f930142cfb3320927936ec8c7 2024-12-09T08:14:40,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/39849e29decd425896a2d192cd53fead to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/39849e29decd425896a2d192cd53fead 2024-12-09T08:14:40,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/966dd50f0a034f2f833b48a432b4320b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/966dd50f0a034f2f833b48a432b4320b 2024-12-09T08:14:40,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a1adfa968a9146158956430f27f5b188 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a1adfa968a9146158956430f27f5b188 2024-12-09T08:14:40,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ba0bbbb1a27247f7a4abfdb156fd11f0 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ba0bbbb1a27247f7a4abfdb156fd11f0 2024-12-09T08:14:40,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/99ba7731daec4e8780fcd68080d52cac to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/99ba7731daec4e8780fcd68080d52cac 2024-12-09T08:14:40,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/f920a8e9c7ad4a858a362a1650c1c501 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/f920a8e9c7ad4a858a362a1650c1c501 2024-12-09T08:14:40,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e622f3285aca4ce792c90431a1154955 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e622f3285aca4ce792c90431a1154955 2024-12-09T08:14:40,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e411022198624a739af6c76e381be9fd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e411022198624a739af6c76e381be9fd 2024-12-09T08:14:40,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/2a899e3c41ea425b8a869f517785f1bc to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/2a899e3c41ea425b8a869f517785f1bc 2024-12-09T08:14:40,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ab44b2262f8848aa989e8fb2cc4e0b5d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/ab44b2262f8848aa989e8fb2cc4e0b5d 2024-12-09T08:14:40,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a81fa37bdb5048a3b64dca6678acb110 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a81fa37bdb5048a3b64dca6678acb110 2024-12-09T08:14:40,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/81aea940785040e0bfdd98aa151e2a98 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/81aea940785040e0bfdd98aa151e2a98 2024-12-09T08:14:40,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a3ee91209661476bb5b5d99c9f61048a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/a3ee91209661476bb5b5d99c9f61048a 2024-12-09T08:14:40,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/146e2e264fde41a1b7fcfa67c383e4f8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/146e2e264fde41a1b7fcfa67c383e4f8 2024-12-09T08:14:40,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bbd0c847d47046d8bc3635114ac3a043 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bbd0c847d47046d8bc3635114ac3a043 2024-12-09T08:14:40,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7fae2f6ecc9d413a8c64655c58a719d4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7fae2f6ecc9d413a8c64655c58a719d4 2024-12-09T08:14:40,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/d538f048a39946d8989782f2394a356f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/d538f048a39946d8989782f2394a356f 2024-12-09T08:14:40,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e4ba0fe7f6c04fe2a376aa62f0f3b290 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/e4ba0fe7f6c04fe2a376aa62f0f3b290 2024-12-09T08:14:40,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/4fde9f6a39bb4dd5930942c771ed92b8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/4fde9f6a39bb4dd5930942c771ed92b8 2024-12-09T08:14:40,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/549569865e3149fd87362c1fed798322 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/549569865e3149fd87362c1fed798322 2024-12-09T08:14:40,398 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/fa320db4850a48a38a4985543c137de5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/fa320db4850a48a38a4985543c137de5 2024-12-09T08:14:40,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/6b3eaf9f9c614256a78b6b689a9c6113 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/6b3eaf9f9c614256a78b6b689a9c6113 2024-12-09T08:14:40,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7aa19197fde44cf593297dab501c462d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/7aa19197fde44cf593297dab501c462d 2024-12-09T08:14:40,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/4baa9144932647c5bd7f6a28caee0afd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/4baa9144932647c5bd7f6a28caee0afd 2024-12-09T08:14:40,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/2f801af8322749f0afc2cd2b8f5bfa1f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/2f801af8322749f0afc2cd2b8f5bfa1f 2024-12-09T08:14:40,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/b170fc7aa1d64df8b95911a79efea616, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/88343ec5de3544fa820fa28cebcf30cb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1cceb52d8ec24e3aad586029082dd846, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f9c889ce7de748b28917afc6445ad0bd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/93e72dc314c24252a633c02e1f5fcdb1, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f44ce88086294119b100e0fe5684ba74, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f7489ba437ac43b48f4cbc316334c53a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/95aed44bf95c439185712790dc15dced, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/7ce78dc04658459c8d0b128d5afe2930, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f3bfce0540fb4dffbbf001e4c77578f5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/862f467d100e408ea1ff9d82075a97da, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/113451ed3a4b4719beccd75a5763c249, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/df72b8e0397f4169b5a14acaaac2f1ca, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/de74ab8169344e46bcbb018bb40b16ee, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1d6b675395f745819f34b87a4ad3324d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/6f289659aea04267b404b7cf360ff969, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/3fa719f12c524cebb5da2bf430ddb587, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/6f7495d3d4b64456b4dd40d55944b471, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/53377803a78f42a8b06ad9230c99b2eb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/704756a15fce49b09dedd2a95b659730, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/d85ddd77b34b42c6a265b1f962b8bdfc, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/211b217f282048609bb9418b0abb8ba6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/c676ca8668b3432cadb28041e33e1c28, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/3b5c66aa94f242c586dbffe24186281f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/2e0f7d7c1465419199e30a500e21a14c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/a98ed9a55b614c7993df3772ded009fa, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/0e624d3d61c94f56a2742addafea417f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/76d572709eea4dcbbe284ecd868445c4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/43fc1eb7024440108ac214040ac3e645, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/4e69b1f9938e4dc3a4dc06cd43240cbc, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/09535b5d328b406791c2c7283247a9ed, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/78ceca4171204137b03638bbc99221d9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1c84a6d4586c4ff1ae06dce35b31a77a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/9eee5ae651a644c2a3ad15f74fcaa890, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/2ec9819c044f49f587e9e7373ccb45ca, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/b8731343d2bd4ac1b7364570de3d3cf8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/c643dcc427d9451da15123cf9757c010, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/ab08c076e880498a9f1a9b51367dd77d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/690805c5b1954330bc232dd7fb762d2c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/0c058a6762ab49c18a7ae7ce7d53b9f7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/25ea71ad080747ee843ffe7b1e89931f] to archive 2024-12-09T08:14:40,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:14:40,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/b170fc7aa1d64df8b95911a79efea616 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/b170fc7aa1d64df8b95911a79efea616 2024-12-09T08:14:40,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/88343ec5de3544fa820fa28cebcf30cb to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/88343ec5de3544fa820fa28cebcf30cb 2024-12-09T08:14:40,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1cceb52d8ec24e3aad586029082dd846 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1cceb52d8ec24e3aad586029082dd846 2024-12-09T08:14:40,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f9c889ce7de748b28917afc6445ad0bd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f9c889ce7de748b28917afc6445ad0bd 2024-12-09T08:14:40,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/93e72dc314c24252a633c02e1f5fcdb1 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/93e72dc314c24252a633c02e1f5fcdb1 2024-12-09T08:14:40,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f44ce88086294119b100e0fe5684ba74 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f44ce88086294119b100e0fe5684ba74 2024-12-09T08:14:40,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f7489ba437ac43b48f4cbc316334c53a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f7489ba437ac43b48f4cbc316334c53a 2024-12-09T08:14:40,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/95aed44bf95c439185712790dc15dced to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/95aed44bf95c439185712790dc15dced 2024-12-09T08:14:40,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/7ce78dc04658459c8d0b128d5afe2930 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/7ce78dc04658459c8d0b128d5afe2930 2024-12-09T08:14:40,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f3bfce0540fb4dffbbf001e4c77578f5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/f3bfce0540fb4dffbbf001e4c77578f5 2024-12-09T08:14:40,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/862f467d100e408ea1ff9d82075a97da to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/862f467d100e408ea1ff9d82075a97da 2024-12-09T08:14:40,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/113451ed3a4b4719beccd75a5763c249 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/113451ed3a4b4719beccd75a5763c249 2024-12-09T08:14:40,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/df72b8e0397f4169b5a14acaaac2f1ca to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/df72b8e0397f4169b5a14acaaac2f1ca 2024-12-09T08:14:40,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/de74ab8169344e46bcbb018bb40b16ee to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/de74ab8169344e46bcbb018bb40b16ee 2024-12-09T08:14:40,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1d6b675395f745819f34b87a4ad3324d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1d6b675395f745819f34b87a4ad3324d 2024-12-09T08:14:40,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/6f289659aea04267b404b7cf360ff969 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/6f289659aea04267b404b7cf360ff969 2024-12-09T08:14:40,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/3fa719f12c524cebb5da2bf430ddb587 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/3fa719f12c524cebb5da2bf430ddb587 2024-12-09T08:14:40,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/6f7495d3d4b64456b4dd40d55944b471 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/6f7495d3d4b64456b4dd40d55944b471 2024-12-09T08:14:40,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/53377803a78f42a8b06ad9230c99b2eb to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/53377803a78f42a8b06ad9230c99b2eb 2024-12-09T08:14:40,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/704756a15fce49b09dedd2a95b659730 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/704756a15fce49b09dedd2a95b659730 2024-12-09T08:14:40,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/d85ddd77b34b42c6a265b1f962b8bdfc to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/d85ddd77b34b42c6a265b1f962b8bdfc 2024-12-09T08:14:40,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/211b217f282048609bb9418b0abb8ba6 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/211b217f282048609bb9418b0abb8ba6 2024-12-09T08:14:40,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/c676ca8668b3432cadb28041e33e1c28 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/c676ca8668b3432cadb28041e33e1c28 2024-12-09T08:14:40,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/3b5c66aa94f242c586dbffe24186281f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/3b5c66aa94f242c586dbffe24186281f 2024-12-09T08:14:40,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/2e0f7d7c1465419199e30a500e21a14c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/2e0f7d7c1465419199e30a500e21a14c 2024-12-09T08:14:40,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/a98ed9a55b614c7993df3772ded009fa to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/a98ed9a55b614c7993df3772ded009fa 2024-12-09T08:14:40,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/0e624d3d61c94f56a2742addafea417f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/0e624d3d61c94f56a2742addafea417f 2024-12-09T08:14:40,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/76d572709eea4dcbbe284ecd868445c4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/76d572709eea4dcbbe284ecd868445c4 2024-12-09T08:14:40,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/43fc1eb7024440108ac214040ac3e645 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/43fc1eb7024440108ac214040ac3e645 2024-12-09T08:14:40,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/4e69b1f9938e4dc3a4dc06cd43240cbc to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/4e69b1f9938e4dc3a4dc06cd43240cbc 2024-12-09T08:14:40,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/09535b5d328b406791c2c7283247a9ed to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/09535b5d328b406791c2c7283247a9ed 2024-12-09T08:14:40,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/78ceca4171204137b03638bbc99221d9 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/78ceca4171204137b03638bbc99221d9 2024-12-09T08:14:40,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1c84a6d4586c4ff1ae06dce35b31a77a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/1c84a6d4586c4ff1ae06dce35b31a77a 2024-12-09T08:14:40,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/9eee5ae651a644c2a3ad15f74fcaa890 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/9eee5ae651a644c2a3ad15f74fcaa890 2024-12-09T08:14:40,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/2ec9819c044f49f587e9e7373ccb45ca to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/2ec9819c044f49f587e9e7373ccb45ca 2024-12-09T08:14:40,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/b8731343d2bd4ac1b7364570de3d3cf8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/b8731343d2bd4ac1b7364570de3d3cf8 2024-12-09T08:14:40,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/c643dcc427d9451da15123cf9757c010 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/c643dcc427d9451da15123cf9757c010 2024-12-09T08:14:40,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/ab08c076e880498a9f1a9b51367dd77d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/ab08c076e880498a9f1a9b51367dd77d 2024-12-09T08:14:40,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/690805c5b1954330bc232dd7fb762d2c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/690805c5b1954330bc232dd7fb762d2c 2024-12-09T08:14:40,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/0c058a6762ab49c18a7ae7ce7d53b9f7 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/0c058a6762ab49c18a7ae7ce7d53b9f7 2024-12-09T08:14:40,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/25ea71ad080747ee843ffe7b1e89931f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/25ea71ad080747ee843ffe7b1e89931f 2024-12-09T08:14:40,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/67d3766a9fad43d887dbcd62ecd9acd3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/105d5dd080844e3e91ef5084b0b099c8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/6abc9fe9495d4ddaabaff11bc3e689d2, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/dfe131f9ff5f492ca1c1ee5176d85947, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/7cec363994554cbe95448910bca60b7c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a5e6e20f1b5d4be59244812394386e8a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/653bf656e5be4d6b86aaf55928a47ae4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/96f15294367a4dc2a106b347d1b93174, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/2ac9d506e36f4b379ed0834ce1e250e5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/b450a3086aca4b2b96cd43d9f2237fcd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a52b60ecf86242949315c3cd9d7c06d1, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/048959e9475249f68b321867abe51398, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/2d8ac1ebe1f34567882b50e6644b29b1, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/64ea61bf5ec9414eaab673903a99fc3f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/67c223cf29dd4beebbc7c9d08994999a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/deb1e7cf2c904138be43ddce51d956d4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/88e4952aedaa4f2f8b47c3be8d0cef92, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/647bb4ac2c0e4f18a2b7e69bb17b63f8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/e565bf0fe6d2495cbda0d641813a0580, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/4d61090df7f242afac304de805fdacf6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ebc702668064f8e8b60a02941b706ca, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/146889fb8a624befb7dd01ababde6de8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8bccaaa842ed4ca087eb9c161add8612, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/5e0f0b8c711648988e24d00729585b21, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a4511dd31ce94132a5ac59cf1c484107, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/44a8dcd5f6cf437c97551ba885c233af, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/5230528ec3c2426bb4b42b4ed36da1ec, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/562ec94dc89447678e6dabfc60ebef42, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/1f1853bbf3bd483b88c9e6a12f066d73, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a660e2d0acdb41029711b4ccf7178c04, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/7bfa3c40f3cf4d078e69e4f1dfe7d6f0, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/64273f2805fb47e98be1acd24a3490e3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ad04f102da04492a4222a32609c4c10, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/092daa5cb5d74f0380832096409614b9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/187a1e67cb664e0ea6607af0cd532f73, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/32c4194b51ab421ab4b5941d52c3d627, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/c91e373a7004467bb934e8b326b51806, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/b7a542f582ce44d1868d5209d9cd8667, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/465f1214ac1546cdb07f25d5130cf258, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/0423dafa91b844aba4dfae8aafec02cf, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/47f057f09bd6454a8cac115f47b84ddb] to archive 2024-12-09T08:14:40,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:14:40,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/67d3766a9fad43d887dbcd62ecd9acd3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/67d3766a9fad43d887dbcd62ecd9acd3 2024-12-09T08:14:40,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/105d5dd080844e3e91ef5084b0b099c8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/105d5dd080844e3e91ef5084b0b099c8 2024-12-09T08:14:40,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/6abc9fe9495d4ddaabaff11bc3e689d2 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/6abc9fe9495d4ddaabaff11bc3e689d2 2024-12-09T08:14:40,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/dfe131f9ff5f492ca1c1ee5176d85947 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/dfe131f9ff5f492ca1c1ee5176d85947 2024-12-09T08:14:40,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/7cec363994554cbe95448910bca60b7c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/7cec363994554cbe95448910bca60b7c 2024-12-09T08:14:40,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a5e6e20f1b5d4be59244812394386e8a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a5e6e20f1b5d4be59244812394386e8a 2024-12-09T08:14:40,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/653bf656e5be4d6b86aaf55928a47ae4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/653bf656e5be4d6b86aaf55928a47ae4 2024-12-09T08:14:40,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/96f15294367a4dc2a106b347d1b93174 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/96f15294367a4dc2a106b347d1b93174 2024-12-09T08:14:40,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/2ac9d506e36f4b379ed0834ce1e250e5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/2ac9d506e36f4b379ed0834ce1e250e5 2024-12-09T08:14:40,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/b450a3086aca4b2b96cd43d9f2237fcd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/b450a3086aca4b2b96cd43d9f2237fcd 2024-12-09T08:14:40,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a52b60ecf86242949315c3cd9d7c06d1 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a52b60ecf86242949315c3cd9d7c06d1 2024-12-09T08:14:40,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/048959e9475249f68b321867abe51398 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/048959e9475249f68b321867abe51398 2024-12-09T08:14:40,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/2d8ac1ebe1f34567882b50e6644b29b1 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/2d8ac1ebe1f34567882b50e6644b29b1 2024-12-09T08:14:40,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/64ea61bf5ec9414eaab673903a99fc3f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/64ea61bf5ec9414eaab673903a99fc3f 2024-12-09T08:14:40,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/67c223cf29dd4beebbc7c9d08994999a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/67c223cf29dd4beebbc7c9d08994999a 2024-12-09T08:14:40,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/deb1e7cf2c904138be43ddce51d956d4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/deb1e7cf2c904138be43ddce51d956d4 2024-12-09T08:14:40,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/88e4952aedaa4f2f8b47c3be8d0cef92 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/88e4952aedaa4f2f8b47c3be8d0cef92 2024-12-09T08:14:40,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/647bb4ac2c0e4f18a2b7e69bb17b63f8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/647bb4ac2c0e4f18a2b7e69bb17b63f8 2024-12-09T08:14:40,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/e565bf0fe6d2495cbda0d641813a0580 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/e565bf0fe6d2495cbda0d641813a0580 2024-12-09T08:14:40,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/4d61090df7f242afac304de805fdacf6 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/4d61090df7f242afac304de805fdacf6 2024-12-09T08:14:40,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ebc702668064f8e8b60a02941b706ca to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ebc702668064f8e8b60a02941b706ca 2024-12-09T08:14:40,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/146889fb8a624befb7dd01ababde6de8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/146889fb8a624befb7dd01ababde6de8 2024-12-09T08:14:40,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8bccaaa842ed4ca087eb9c161add8612 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8bccaaa842ed4ca087eb9c161add8612 2024-12-09T08:14:40,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/5e0f0b8c711648988e24d00729585b21 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/5e0f0b8c711648988e24d00729585b21 2024-12-09T08:14:40,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a4511dd31ce94132a5ac59cf1c484107 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a4511dd31ce94132a5ac59cf1c484107 2024-12-09T08:14:40,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/44a8dcd5f6cf437c97551ba885c233af to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/44a8dcd5f6cf437c97551ba885c233af 2024-12-09T08:14:40,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/5230528ec3c2426bb4b42b4ed36da1ec to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/5230528ec3c2426bb4b42b4ed36da1ec 2024-12-09T08:14:40,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/562ec94dc89447678e6dabfc60ebef42 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/562ec94dc89447678e6dabfc60ebef42 2024-12-09T08:14:40,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/1f1853bbf3bd483b88c9e6a12f066d73 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/1f1853bbf3bd483b88c9e6a12f066d73 2024-12-09T08:14:40,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a660e2d0acdb41029711b4ccf7178c04 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/a660e2d0acdb41029711b4ccf7178c04 2024-12-09T08:14:40,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/7bfa3c40f3cf4d078e69e4f1dfe7d6f0 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/7bfa3c40f3cf4d078e69e4f1dfe7d6f0 2024-12-09T08:14:40,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/64273f2805fb47e98be1acd24a3490e3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/64273f2805fb47e98be1acd24a3490e3 2024-12-09T08:14:40,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ad04f102da04492a4222a32609c4c10 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ad04f102da04492a4222a32609c4c10 2024-12-09T08:14:40,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/092daa5cb5d74f0380832096409614b9 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/092daa5cb5d74f0380832096409614b9 2024-12-09T08:14:40,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/187a1e67cb664e0ea6607af0cd532f73 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/187a1e67cb664e0ea6607af0cd532f73 2024-12-09T08:14:40,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/32c4194b51ab421ab4b5941d52c3d627 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/32c4194b51ab421ab4b5941d52c3d627 2024-12-09T08:14:40,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/c91e373a7004467bb934e8b326b51806 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/c91e373a7004467bb934e8b326b51806 2024-12-09T08:14:40,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/b7a542f582ce44d1868d5209d9cd8667 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/b7a542f582ce44d1868d5209d9cd8667 2024-12-09T08:14:40,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/465f1214ac1546cdb07f25d5130cf258 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/465f1214ac1546cdb07f25d5130cf258 2024-12-09T08:14:40,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/0423dafa91b844aba4dfae8aafec02cf to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/0423dafa91b844aba4dfae8aafec02cf 2024-12-09T08:14:40,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/47f057f09bd6454a8cac115f47b84ddb to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/47f057f09bd6454a8cac115f47b84ddb 2024-12-09T08:14:40,511 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/recovered.edits/636.seqid, newMaxSeqId=636, maxSeqId=1 2024-12-09T08:14:40,512 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. 2024-12-09T08:14:40,512 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 589b933f5d38176578d90952a67fb0a1: Waiting for close lock at 1733732079012Running coprocessor pre-close hooks at 1733732079012Disabling compacts and flushes for region at 1733732079013 (+1 ms)Disabling writes for close at 1733732079013Obtaining lock to block concurrent updates at 1733732079013Preparing flush snapshotting stores in 589b933f5d38176578d90952a67fb0a1 at 1733732079013Finished memstore snapshotting TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733732079014 (+1 ms)Flushing stores of TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1. at 1733732079015 (+1 ms)Flushing 589b933f5d38176578d90952a67fb0a1/A: creating writer at 1733732079015Flushing 589b933f5d38176578d90952a67fb0a1/A: appending metadata at 1733732079020 (+5 ms)Flushing 589b933f5d38176578d90952a67fb0a1/A: closing flushed file at 1733732079020Flushing 589b933f5d38176578d90952a67fb0a1/B: creating writer at 1733732079436 (+416 ms)Flushing 589b933f5d38176578d90952a67fb0a1/B: appending metadata at 1733732079440 (+4 ms)Flushing 589b933f5d38176578d90952a67fb0a1/B: closing flushed file at 1733732079440Flushing 589b933f5d38176578d90952a67fb0a1/C: creating writer at 1733732079905 (+465 ms)Flushing 589b933f5d38176578d90952a67fb0a1/C: appending metadata at 1733732079909 (+4 ms)Flushing 589b933f5d38176578d90952a67fb0a1/C: closing flushed file at 1733732079909Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74907e5: reopening flushed file at 1733732080325 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b795830: reopening flushed file at 1733732080334 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c08b485: reopening flushed file at 1733732080340 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 589b933f5d38176578d90952a67fb0a1 in 1332ms, sequenceid=633, compaction requested=false at 1733732080346 (+6 ms)Writing region close event to WAL at 1733732080508 (+162 ms)Running coprocessor post-close hooks at 1733732080511 (+3 ms)Closed at 1733732080511 2024-12-09T08:14:40,514 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:40,515 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=589b933f5d38176578d90952a67fb0a1, regionState=CLOSED 2024-12-09T08:14:40,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:14:40,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-09T08:14:40,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 589b933f5d38176578d90952a67fb0a1, server=5bda946059d1,43255,1733732044053 in 1.6670 sec 2024-12-09T08:14:40,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-09T08:14:40,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=589b933f5d38176578d90952a67fb0a1, UNASSIGN in 1.6730 sec 2024-12-09T08:14:40,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-09T08:14:40,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6790 sec 2024-12-09T08:14:40,524 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732080524"}]},"ts":"1733732080524"} 2024-12-09T08:14:40,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T08:14:40,526 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T08:14:40,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6930 sec 2024-12-09T08:14:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T08:14:40,983 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:40,991 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T08:14:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:14:41,000 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:14:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T08:14:41,001 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:14:41,006 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:41,012 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/recovered.edits] 2024-12-09T08:14:41,016 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/b3ee14dc88bb4510aa5fba5af32fe560 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/b3ee14dc88bb4510aa5fba5af32fe560 2024-12-09T08:14:41,017 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bdedc0e8ac274e4a9be2d3c79e72d126 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/A/bdedc0e8ac274e4a9be2d3c79e72d126 2024-12-09T08:14:41,020 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/7df9a35684f64cb1aa7ecd27addbbb99 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/7df9a35684f64cb1aa7ecd27addbbb99 2024-12-09T08:14:41,021 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/8fe1e278d1c844e5b179049a0640e09f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/B/8fe1e278d1c844e5b179049a0640e09f 2024-12-09T08:14:41,024 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/491b111fddf34df5b57b31b2460e774d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/491b111fddf34df5b57b31b2460e774d 2024-12-09T08:14:41,026 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ed9f7e43ad741a490e22c196a56d03b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/C/8ed9f7e43ad741a490e22c196a56d03b 2024-12-09T08:14:41,028 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/recovered.edits/636.seqid to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1/recovered.edits/636.seqid 2024-12-09T08:14:41,028 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/589b933f5d38176578d90952a67fb0a1 2024-12-09T08:14:41,029 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T08:14:41,033 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:14:41,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-09T08:14:41,041 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T08:14:41,068 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T08:14:41,069 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:14:41,070 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T08:14:41,070 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733732081070"}]},"ts":"9223372036854775807"} 2024-12-09T08:14:41,074 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T08:14:41,074 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 589b933f5d38176578d90952a67fb0a1, NAME => 'TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T08:14:41,074 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T08:14:41,074 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733732081074"}]},"ts":"9223372036854775807"} 2024-12-09T08:14:41,077 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T08:14:41,078 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:14:41,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-12-09T08:14:41,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T08:14:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T08:14:41,112 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:41,121 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;5bda946059d1:43255-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/5bda946059d1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_646891914_22 at /127.0.0.1:41252 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=386 (was 264) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4024 (was 4645) 2024-12-09T08:14:41,130 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=386, ProcessCount=11, AvailableMemoryMB=4024 2024-12-09T08:14:41,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T08:14:41,132 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:14:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T08:14:41,135 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T08:14:41,135 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:41,135 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-12-09T08:14:41,136 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T08:14:41,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T08:14:41,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741965_1141 (size=1061) 2024-12-09T08:14:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T08:14:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T08:14:41,548 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447 2024-12-09T08:14:41,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741966_1142 (size=53) 2024-12-09T08:14:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T08:14:41,961 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:14:41,962 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c9202617e4fb01f766f444f5e419c460, disabling compactions & flushes 2024-12-09T08:14:41,962 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:41,962 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:41,962 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. after waiting 0 ms 2024-12-09T08:14:41,962 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:41,962 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:41,962 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c9202617e4fb01f766f444f5e419c460: Waiting for close lock at 1733732081962Disabling compacts and flushes for region at 1733732081962Disabling writes for close at 1733732081962Writing region close event to WAL at 1733732081962Closed at 1733732081962 2024-12-09T08:14:41,963 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T08:14:41,964 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733732081963"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733732081963"}]},"ts":"1733732081963"} 2024-12-09T08:14:41,966 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T08:14:41,967 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T08:14:41,967 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732081967"}]},"ts":"1733732081967"} 2024-12-09T08:14:41,970 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T08:14:41,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, ASSIGN}] 2024-12-09T08:14:41,972 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, ASSIGN 2024-12-09T08:14:41,973 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, ASSIGN; state=OFFLINE, location=5bda946059d1,43255,1733732044053; forceNewPlan=false, retain=false 2024-12-09T08:14:42,124 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=c9202617e4fb01f766f444f5e419c460, regionState=OPENING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:14:42,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, ASSIGN because future has completed 2024-12-09T08:14:42,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:14:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T08:14:42,284 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:42,285 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:14:42,285 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,285 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:14:42,286 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,286 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,288 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,290 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:14:42,292 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9202617e4fb01f766f444f5e419c460 columnFamilyName A 2024-12-09T08:14:42,292 DEBUG [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:42,293 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.HStore(327): Store=c9202617e4fb01f766f444f5e419c460/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:14:42,293 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,294 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:14:42,295 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9202617e4fb01f766f444f5e419c460 columnFamilyName B 2024-12-09T08:14:42,295 DEBUG [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:42,295 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.HStore(327): Store=c9202617e4fb01f766f444f5e419c460/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:14:42,295 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,296 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:14:42,297 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9202617e4fb01f766f444f5e419c460 columnFamilyName C 2024-12-09T08:14:42,297 DEBUG [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:42,297 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.HStore(327): Store=c9202617e4fb01f766f444f5e419c460/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:14:42,297 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:42,297 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,298 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,298 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,300 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,300 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,300 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:14:42,301 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,303 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:14:42,303 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened c9202617e4fb01f766f444f5e419c460; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70105614, jitterRate=0.044655054807662964}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:14:42,303 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:42,304 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for c9202617e4fb01f766f444f5e419c460: Running coprocessor pre-open hook at 1733732082286Writing region info on filesystem at 1733732082286Initializing all the Stores at 1733732082287 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732082287Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732082288 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732082288Cleaning up temporary data from old regions at 1733732082300 (+12 ms)Running coprocessor post-open hooks at 1733732082303 (+3 ms)Region opened successfully at 1733732082304 (+1 ms) 2024-12-09T08:14:42,305 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., pid=32, masterSystemTime=1733732082279 2024-12-09T08:14:42,306 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:42,306 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:42,307 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=c9202617e4fb01f766f444f5e419c460, regionState=OPEN, openSeqNum=2, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:14:42,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:14:42,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-09T08:14:42,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 in 182 msec 2024-12-09T08:14:42,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-09T08:14:42,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, ASSIGN in 341 msec 2024-12-09T08:14:42,314 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T08:14:42,314 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732082314"}]},"ts":"1733732082314"} 2024-12-09T08:14:42,316 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T08:14:42,317 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T08:14:42,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-12-09T08:14:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T08:14:43,282 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:43,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:43,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:43,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:43,289 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:43,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:43,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16525882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:43,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:43,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:43,295 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e686d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:43,296 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:43,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:43,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:43,300 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T08:14:43,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 5bda946059d1,45611,1733732043366 2024-12-09T08:14:43,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@29fa5185 2024-12-09T08:14:43,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T08:14:43,302 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T08:14:43,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T08:14:43,309 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:14:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T08:14:43,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741967_1143 (size=1097) 2024-12-09T08:14:43,730 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T08:14:43,730 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T08:14:43,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T08:14:43,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, REOPEN/MOVE}] 2024-12-09T08:14:43,740 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, REOPEN/MOVE 2024-12-09T08:14:43,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c9202617e4fb01f766f444f5e419c460, regionState=CLOSING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:14:43,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, REOPEN/MOVE because future has completed 2024-12-09T08:14:43,742 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:14:43,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:14:43,897 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:43,897 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:14:43,897 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing c9202617e4fb01f766f444f5e419c460, disabling compactions & flushes 2024-12-09T08:14:43,898 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:43,898 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:43,898 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. after waiting 0 ms 2024-12-09T08:14:43,898 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:43,906 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T08:14:43,907 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:43,907 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for c9202617e4fb01f766f444f5e419c460: Waiting for close lock at 1733732083897Running coprocessor pre-close hooks at 1733732083897Disabling compacts and flushes for region at 1733732083897Disabling writes for close at 1733732083898 (+1 ms)Writing region close event to WAL at 1733732083899 (+1 ms)Running coprocessor post-close hooks at 1733732083907 (+8 ms)Closed at 1733732083907 2024-12-09T08:14:43,907 WARN [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: c9202617e4fb01f766f444f5e419c460 to self. 2024-12-09T08:14:43,909 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:43,910 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c9202617e4fb01f766f444f5e419c460, regionState=CLOSED 2024-12-09T08:14:43,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:14:43,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-09T08:14:43,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 in 170 msec 2024-12-09T08:14:43,914 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, REOPEN/MOVE; state=CLOSED, location=5bda946059d1,43255,1733732044053; forceNewPlan=false, retain=true 2024-12-09T08:14:44,065 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c9202617e4fb01f766f444f5e419c460, regionState=OPENING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, REOPEN/MOVE because future has completed 2024-12-09T08:14:44,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:14:44,233 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:44,234 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:14:44,235 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,235 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:14:44,236 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,236 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,241 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,242 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:14:44,249 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9202617e4fb01f766f444f5e419c460 columnFamilyName A 2024-12-09T08:14:44,251 DEBUG [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:44,251 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.HStore(327): Store=c9202617e4fb01f766f444f5e419c460/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:14:44,252 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,252 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:14:44,253 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9202617e4fb01f766f444f5e419c460 columnFamilyName B 2024-12-09T08:14:44,253 DEBUG [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:44,253 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.HStore(327): Store=c9202617e4fb01f766f444f5e419c460/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:14:44,253 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,254 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:14:44,254 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9202617e4fb01f766f444f5e419c460 columnFamilyName C 2024-12-09T08:14:44,254 DEBUG [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:44,255 INFO [StoreOpener-c9202617e4fb01f766f444f5e419c460-1 {}] regionserver.HStore(327): Store=c9202617e4fb01f766f444f5e419c460/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:14:44,255 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:44,255 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,256 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,257 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,258 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,258 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,258 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:14:44,260 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,261 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened c9202617e4fb01f766f444f5e419c460; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71511164, jitterRate=0.06559938192367554}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:14:44,261 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,263 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for c9202617e4fb01f766f444f5e419c460: Running coprocessor pre-open hook at 1733732084236Writing region info on filesystem at 1733732084236Initializing all the Stores at 1733732084237 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732084238 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732084238Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732084238Cleaning up temporary data from old regions at 1733732084258 (+20 ms)Running coprocessor post-open hooks at 1733732084261 (+3 ms)Region opened successfully at 1733732084263 (+2 ms) 2024-12-09T08:14:44,264 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., pid=37, masterSystemTime=1733732084224 2024-12-09T08:14:44,266 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:44,266 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:44,267 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c9202617e4fb01f766f444f5e419c460, regionState=OPEN, openSeqNum=5, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:14:44,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-12-09T08:14:44,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 in 198 msec 2024-12-09T08:14:44,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-09T08:14:44,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, REOPEN/MOVE in 532 msec 2024-12-09T08:14:44,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-09T08:14:44,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 542 msec 2024-12-09T08:14:44,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 966 msec 2024-12-09T08:14:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-09T08:14:44,281 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:44,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:44,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:44,283 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@664c92ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:44,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:44,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:44,285 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:44,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180c6c34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:44,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:44,287 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:44,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@273813dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:44,289 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:44,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aacbeb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:44,291 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:44,291 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f908f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:44,293 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:44,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@602f29a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:44,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:44,295 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a939a90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:44,297 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4c1db4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:44,298 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:44,299 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:44,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c976095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:44,301 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d590114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:44,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:44,304 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2646de50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:44,306 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d5f61e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,307 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:44,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:44,307 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aed0cf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:44,309 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169788b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:44,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:44,311 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3764eaed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:44,313 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e62cbdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:14:44,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:14:44,315 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:14:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:14:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:14:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c806f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:14:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:14:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:14:44,316 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:14:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d35809d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:14:44,318 DEBUG [Thread-713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:44,318 DEBUG [Thread-706 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:44,318 DEBUG [Thread-719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:44,318 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:44,318 DEBUG [Thread-715 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:44,319 DEBUG [Thread-717 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:44,319 DEBUG [Thread-710 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:44,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:44,320 DEBUG [Thread-704 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-09T08:14:44,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:44,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:44,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:44,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:44,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:44,322 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T08:14:44,324 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:44,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:44,324 DEBUG [Thread-704 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:44,325 DEBUG [Thread-704 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:44,325 DEBUG [Thread-708 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:14:44,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:14:44,329 DEBUG [Thread-706 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:44,329 DEBUG [Thread-702 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:14:44,329 DEBUG [Thread-706 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:14:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:14:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:44,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:14:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:14:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:14:44,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:14:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:14:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:14:44,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:14:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:14:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:14:44,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:14:44,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:44,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:14:44,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:44,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:14:44,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:44,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e181e1a33997410f9aa51038cad15cc0_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732084343/Put/seqid=0 2024-12-09T08:14:44,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732144387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733732084377, max timestamp=1733732084390Num uniques 859; 2024-12-09T08:14:44,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733732084377, max timestamp=1733732084391Num uniques 859; 2024-12-09T08:14:44,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T08:14:44,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T08:14:44,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733732084377, max timestamp=1733732084391Num uniques 859; 2024-12-09T08:14:44,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T08:14:44,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741968_1144 (size=14594) 2024-12-09T08:14:44,410 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:44,417 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e181e1a33997410f9aa51038cad15cc0_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e181e1a33997410f9aa51038cad15cc0_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,419 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/a40371302fad4c26b54dc1a954a49b87, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:44,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/a40371302fad4c26b54dc1a954a49b87 is 175, key is test_row_0/A:col10/1733732084343/Put/seqid=0 2024-12-09T08:14:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T08:14:44,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732144433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732144434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732144435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732144436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741969_1145 (size=39549) 2024-12-09T08:14:44,467 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/a40371302fad4c26b54dc1a954a49b87 2024-12-09T08:14:44,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T08:14:44,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:44,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:44,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:44,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:44,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:44,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/bd7914d17c8e4626adb2a9074616ffdc is 50, key is test_row_0/B:col10/1733732084343/Put/seqid=0 2024-12-09T08:14:44,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732144501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741970_1146 (size=12001) 2024-12-09T08:14:44,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/bd7914d17c8e4626adb2a9074616ffdc 2024-12-09T08:14:44,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732144547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732144552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732144552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732144552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/8e71d0ba91ef448bbe8d0f84c015ebcf is 50, key is test_row_0/C:col10/1733732084343/Put/seqid=0 2024-12-09T08:14:44,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741971_1147 (size=12001) 2024-12-09T08:14:44,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/8e71d0ba91ef448bbe8d0f84c015ebcf 2024-12-09T08:14:44,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/a40371302fad4c26b54dc1a954a49b87 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/a40371302fad4c26b54dc1a954a49b87 2024-12-09T08:14:44,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/a40371302fad4c26b54dc1a954a49b87, entries=200, sequenceid=14, filesize=38.6 K 2024-12-09T08:14:44,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/bd7914d17c8e4626adb2a9074616ffdc as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/bd7914d17c8e4626adb2a9074616ffdc 2024-12-09T08:14:44,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/bd7914d17c8e4626adb2a9074616ffdc, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T08:14:44,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/8e71d0ba91ef448bbe8d0f84c015ebcf as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/8e71d0ba91ef448bbe8d0f84c015ebcf 2024-12-09T08:14:44,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/8e71d0ba91ef448bbe8d0f84c015ebcf, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T08:14:44,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c9202617e4fb01f766f444f5e419c460 in 267ms, sequenceid=14, compaction requested=false 2024-12-09T08:14:44,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:44,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T08:14:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:44,637 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-09T08:14:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:14:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:14:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:14:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T08:14:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091bfae47908864dbcb8124f08d72d3191_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732084433/Put/seqid=0 2024-12-09T08:14:44,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741972_1148 (size=12154) 2024-12-09T08:14:44,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:44,700 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091bfae47908864dbcb8124f08d72d3191_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091bfae47908864dbcb8124f08d72d3191_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/f5b9cb2018c54ba791875f03f5ecf677, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:44,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/f5b9cb2018c54ba791875f03f5ecf677 is 175, key is test_row_0/A:col10/1733732084433/Put/seqid=0 2024-12-09T08:14:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:44,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:44,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741973_1149 (size=30955) 2024-12-09T08:14:44,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732144757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732144762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732144762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732144771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732144771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732144871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T08:14:45,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732145071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732145071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732145082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732145082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732145094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,115 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/f5b9cb2018c54ba791875f03f5ecf677 2024-12-09T08:14:45,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/ca8c0741a39149e5bc6a63b306bbec5c is 50, key is test_row_0/B:col10/1733732084433/Put/seqid=0 2024-12-09T08:14:45,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741974_1150 (size=12001) 2024-12-09T08:14:45,134 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/ca8c0741a39149e5bc6a63b306bbec5c 2024-12-09T08:14:45,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/146dbd5b557c490293c919b52d0a0837 is 50, key is test_row_0/C:col10/1733732084433/Put/seqid=0 2024-12-09T08:14:45,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741975_1151 (size=12001) 2024-12-09T08:14:45,157 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/146dbd5b557c490293c919b52d0a0837 2024-12-09T08:14:45,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/f5b9cb2018c54ba791875f03f5ecf677 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f5b9cb2018c54ba791875f03f5ecf677 2024-12-09T08:14:45,183 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f5b9cb2018c54ba791875f03f5ecf677, entries=150, sequenceid=49, filesize=30.2 K 2024-12-09T08:14:45,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/ca8c0741a39149e5bc6a63b306bbec5c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/ca8c0741a39149e5bc6a63b306bbec5c 2024-12-09T08:14:45,189 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/ca8c0741a39149e5bc6a63b306bbec5c, entries=150, sequenceid=49, filesize=11.7 K 2024-12-09T08:14:45,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/146dbd5b557c490293c919b52d0a0837 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/146dbd5b557c490293c919b52d0a0837 2024-12-09T08:14:45,197 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/146dbd5b557c490293c919b52d0a0837, entries=150, sequenceid=49, filesize=11.7 K 2024-12-09T08:14:45,198 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for c9202617e4fb01f766f444f5e419c460 in 561ms, sequenceid=49, compaction requested=false 2024-12-09T08:14:45,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:45,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:45,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-09T08:14:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-09T08:14:45,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-09T08:14:45,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 876 msec 2024-12-09T08:14:45,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 884 msec 2024-12-09T08:14:45,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:14:45,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:14:45,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:45,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:14:45,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:45,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:14:45,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:45,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c7090a33cc524baa82325770ded3d894_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732084739/Put/seqid=0 2024-12-09T08:14:45,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741976_1152 (size=12154) 2024-12-09T08:14:45,427 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:45,434 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c7090a33cc524baa82325770ded3d894_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c7090a33cc524baa82325770ded3d894_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:45,436 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/4082411b22c647499dac3f6d92bfd743, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:45,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/4082411b22c647499dac3f6d92bfd743 is 175, key is test_row_0/A:col10/1733732084739/Put/seqid=0 2024-12-09T08:14:45,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741977_1153 (size=30955) 2024-12-09T08:14:45,448 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/4082411b22c647499dac3f6d92bfd743 2024-12-09T08:14:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T08:14:45,462 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:45,462 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-09T08:14:45,465 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T08:14:45,466 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:45,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:45,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/37eaaa72371b4c2b811d713640623e45 is 50, key is test_row_0/B:col10/1733732084739/Put/seqid=0 2024-12-09T08:14:45,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741978_1154 (size=12001) 2024-12-09T08:14:45,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/37eaaa72371b4c2b811d713640623e45 2024-12-09T08:14:45,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/c8e9c17f51aa4eda8aba99cf87181607 is 50, key is test_row_0/C:col10/1733732084739/Put/seqid=0 2024-12-09T08:14:45,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732085465, max timestamp=1733732085529Num uniques 859; 2024-12-09T08:14:45,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732085465, max timestamp=1733732085529Num uniques 859; 2024-12-09T08:14:45,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:14:45,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:14:45,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732085494, max timestamp=1733732085529Num uniques 859; 2024-12-09T08:14:45,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:14:45,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741979_1155 (size=12001) 2024-12-09T08:14:45,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/c8e9c17f51aa4eda8aba99cf87181607 2024-12-09T08:14:45,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/4082411b22c647499dac3f6d92bfd743 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/4082411b22c647499dac3f6d92bfd743 2024-12-09T08:14:45,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/4082411b22c647499dac3f6d92bfd743, entries=150, sequenceid=60, filesize=30.2 K 2024-12-09T08:14:45,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/37eaaa72371b4c2b811d713640623e45 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/37eaaa72371b4c2b811d713640623e45 2024-12-09T08:14:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T08:14:45,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/37eaaa72371b4c2b811d713640623e45, entries=150, sequenceid=60, filesize=11.7 K 2024-12-09T08:14:45,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/c8e9c17f51aa4eda8aba99cf87181607 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c8e9c17f51aa4eda8aba99cf87181607 2024-12-09T08:14:45,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c8e9c17f51aa4eda8aba99cf87181607, entries=150, sequenceid=60, filesize=11.7 K 2024-12-09T08:14:45,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=91.69 KB/93890 for c9202617e4fb01f766f444f5e419c460 in 195ms, sequenceid=60, compaction requested=true 2024-12-09T08:14:45,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:45,590 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:45,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:45,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:45,590 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:45,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:45,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:45,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:45,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:45,592 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:45,592 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:45,592 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/A is initiating minor compaction (all files) 2024-12-09T08:14:45,592 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/B is initiating minor compaction (all files) 2024-12-09T08:14:45,592 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/A in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:45,592 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/B in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:45,592 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/bd7914d17c8e4626adb2a9074616ffdc, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/ca8c0741a39149e5bc6a63b306bbec5c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/37eaaa72371b4c2b811d713640623e45] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=35.2 K 2024-12-09T08:14:45,592 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/a40371302fad4c26b54dc1a954a49b87, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f5b9cb2018c54ba791875f03f5ecf677, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/4082411b22c647499dac3f6d92bfd743] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=99.1 K 2024-12-09T08:14:45,592 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:45,593 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting bd7914d17c8e4626adb2a9074616ffdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733732084340 2024-12-09T08:14:45,593 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/a40371302fad4c26b54dc1a954a49b87, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f5b9cb2018c54ba791875f03f5ecf677, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/4082411b22c647499dac3f6d92bfd743] 2024-12-09T08:14:45,593 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ca8c0741a39149e5bc6a63b306bbec5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733732084421 2024-12-09T08:14:45,593 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a40371302fad4c26b54dc1a954a49b87, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733732084338 2024-12-09T08:14:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:45,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=105.11 KB heapSize=295.53 KB 2024-12-09T08:14:45,593 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 37eaaa72371b4c2b811d713640623e45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733732084739 2024-12-09T08:14:45,593 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5b9cb2018c54ba791875f03f5ecf677, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733732084421 2024-12-09T08:14:45,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:14:45,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:45,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:14:45,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:45,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:14:45,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:14:45,595 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4082411b22c647499dac3f6d92bfd743, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733732084739 2024-12-09T08:14:45,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096f183abc19ed4035a1ffa0b7e1de523a_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732085582/Put/seqid=0 2024-12-09T08:14:45,608 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:45,615 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#B#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:45,616 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/2662b2ae502e417380c272f410bc3fde is 50, key is test_row_0/B:col10/1733732084739/Put/seqid=0 2024-12-09T08:14:45,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T08:14:45,622 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412090efdcea6f1d5474e9607d84ec9338c3d_c9202617e4fb01f766f444f5e419c460 store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:45,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:45,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:45,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:45,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:45,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741980_1156 (size=12154) 2024-12-09T08:14:45,630 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412090efdcea6f1d5474e9607d84ec9338c3d_c9202617e4fb01f766f444f5e419c460, store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:45,630 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090efdcea6f1d5474e9607d84ec9338c3d_c9202617e4fb01f766f444f5e419c460 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:45,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732145628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732145629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732145631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732145632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741981_1157 (size=12104) 2024-12-09T08:14:45,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732145634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,649 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/2662b2ae502e417380c272f410bc3fde as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/2662b2ae502e417380c272f410bc3fde 2024-12-09T08:14:45,657 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9202617e4fb01f766f444f5e419c460/B of c9202617e4fb01f766f444f5e419c460 into 2662b2ae502e417380c272f410bc3fde(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:45,657 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:45,657 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/B, priority=13, startTime=1733732085590; duration=0sec 2024-12-09T08:14:45,657 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:45,657 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:B 2024-12-09T08:14:45,658 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:45,659 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:45,659 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/C is initiating minor compaction (all files) 2024-12-09T08:14:45,659 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/C in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:45,659 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/8e71d0ba91ef448bbe8d0f84c015ebcf, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/146dbd5b557c490293c919b52d0a0837, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c8e9c17f51aa4eda8aba99cf87181607] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=35.2 K 2024-12-09T08:14:45,659 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e71d0ba91ef448bbe8d0f84c015ebcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733732084340 2024-12-09T08:14:45,660 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 146dbd5b557c490293c919b52d0a0837, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733732084421 2024-12-09T08:14:45,660 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting c8e9c17f51aa4eda8aba99cf87181607, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733732084739 2024-12-09T08:14:45,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741982_1158 (size=4469) 2024-12-09T08:14:45,670 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#A#compaction#138 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:45,671 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/d2d5fc177a124a418267376765374b98 is 175, key is test_row_0/A:col10/1733732084739/Put/seqid=0 2024-12-09T08:14:45,687 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:45,688 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/887aa7927af44cba9b20348844a9c036 is 50, key is test_row_0/C:col10/1733732084739/Put/seqid=0 2024-12-09T08:14:45,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741983_1159 (size=31058) 2024-12-09T08:14:45,700 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/d2d5fc177a124a418267376765374b98 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d2d5fc177a124a418267376765374b98 2024-12-09T08:14:45,708 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9202617e4fb01f766f444f5e419c460/A of c9202617e4fb01f766f444f5e419c460 into d2d5fc177a124a418267376765374b98(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:45,708 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:45,708 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/A, priority=13, startTime=1733732085590; duration=0sec 2024-12-09T08:14:45,708 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:45,708 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:A 2024-12-09T08:14:45,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741984_1160 (size=12104) 2024-12-09T08:14:45,720 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/887aa7927af44cba9b20348844a9c036 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/887aa7927af44cba9b20348844a9c036 2024-12-09T08:14:45,728 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9202617e4fb01f766f444f5e419c460/C of c9202617e4fb01f766f444f5e419c460 into 887aa7927af44cba9b20348844a9c036(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:45,728 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:45,728 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/C, priority=13, startTime=1733732085590; duration=0sec 2024-12-09T08:14:45,728 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:45,728 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:C 2024-12-09T08:14:45,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732145742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732145742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732145751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732145751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732145751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T08:14:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:45,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T08:14:45,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T08:14:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:45,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:45,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:45,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:45,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:45,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732145951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732145951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732145961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732145962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732145962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,028 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:46,032 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096f183abc19ed4035a1ffa0b7e1de523a_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096f183abc19ed4035a1ffa0b7e1de523a_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:46,032 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e603db79e3a04427bfbcf4f787404c9b, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:46,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e603db79e3a04427bfbcf4f787404c9b is 175, key is test_row_0/A:col10/1733732085582/Put/seqid=0 2024-12-09T08:14:46,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741985_1161 (size=30955) 2024-12-09T08:14:46,069 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:14:46,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T08:14:46,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:46,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:46,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:46,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:46,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T08:14:46,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T08:14:46,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:46,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:46,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:46,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:46,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:46,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732146261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732146262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732146271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732146271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732146271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T08:14:46,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:46,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:46,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:46,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:46,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:46,437 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e603db79e3a04427bfbcf4f787404c9b 2024-12-09T08:14:46,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/151e4c99ba044393a1450b52e0acaeab is 50, key is test_row_0/B:col10/1733732085582/Put/seqid=0 2024-12-09T08:14:46,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741986_1162 (size=12001) 2024-12-09T08:14:46,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/151e4c99ba044393a1450b52e0acaeab 2024-12-09T08:14:46,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/2b48ecde64af4cdb9cd6fb25a82bc9af is 50, key is test_row_0/C:col10/1733732085582/Put/seqid=0 2024-12-09T08:14:46,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741987_1163 (size=12001) 2024-12-09T08:14:46,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/2b48ecde64af4cdb9cd6fb25a82bc9af 2024-12-09T08:14:46,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e603db79e3a04427bfbcf4f787404c9b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e603db79e3a04427bfbcf4f787404c9b 2024-12-09T08:14:46,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e603db79e3a04427bfbcf4f787404c9b, entries=150, sequenceid=92, filesize=30.2 K 2024-12-09T08:14:46,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/151e4c99ba044393a1450b52e0acaeab as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/151e4c99ba044393a1450b52e0acaeab 2024-12-09T08:14:46,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/151e4c99ba044393a1450b52e0acaeab, entries=150, sequenceid=92, filesize=11.7 K 2024-12-09T08:14:46,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/2b48ecde64af4cdb9cd6fb25a82bc9af as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/2b48ecde64af4cdb9cd6fb25a82bc9af 2024-12-09T08:14:46,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/2b48ecde64af4cdb9cd6fb25a82bc9af, entries=150, sequenceid=92, filesize=11.7 K 2024-12-09T08:14:46,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~105.11 KB/107630, heapSize ~295.48 KB/302576, currentSize=73.80 KB/75570 for c9202617e4fb01f766f444f5e419c460 in 943ms, sequenceid=92, compaction requested=false 2024-12-09T08:14:46,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:46,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T08:14:46,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:46,546 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T08:14:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:14:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:14:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:14:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:46,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e2e73e0a97884db196c3bc95027b65d3_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732085628/Put/seqid=0 2024-12-09T08:14:46,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741988_1164 (size=12154) 2024-12-09T08:14:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T08:14:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:46,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:46,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732146788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732146790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732146792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732146793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732146795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732146901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732146901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732146902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732146902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732146902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:46,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:46,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:46,970 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e2e73e0a97884db196c3bc95027b65d3_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e2e73e0a97884db196c3bc95027b65d3_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:46,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/77c64fac696a412c953f3ad83018452a, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:46,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/77c64fac696a412c953f3ad83018452a is 175, key is test_row_0/A:col10/1733732085628/Put/seqid=0 2024-12-09T08:14:46,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741989_1165 (size=30955) 2024-12-09T08:14:47,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732147112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732147112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732147112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732147112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732147112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,378 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/77c64fac696a412c953f3ad83018452a 2024-12-09T08:14:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/18d61690743b4d218de0c5ae9e49204b is 50, key is test_row_0/B:col10/1733732085628/Put/seqid=0 2024-12-09T08:14:47,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741990_1166 (size=12001) 2024-12-09T08:14:47,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732147421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732147421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732147422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732147422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732147422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T08:14:47,792 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/18d61690743b4d218de0c5ae9e49204b 2024-12-09T08:14:47,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/4b4dcb65f0c94e9092ad07680dc4d42a is 50, key is test_row_0/C:col10/1733732085628/Put/seqid=0 2024-12-09T08:14:47,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741991_1167 (size=12001) 2024-12-09T08:14:47,811 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/4b4dcb65f0c94e9092ad07680dc4d42a 2024-12-09T08:14:47,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/77c64fac696a412c953f3ad83018452a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/77c64fac696a412c953f3ad83018452a 2024-12-09T08:14:47,821 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/77c64fac696a412c953f3ad83018452a, entries=150, sequenceid=109, filesize=30.2 K 2024-12-09T08:14:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/18d61690743b4d218de0c5ae9e49204b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/18d61690743b4d218de0c5ae9e49204b 2024-12-09T08:14:47,825 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/18d61690743b4d218de0c5ae9e49204b, entries=150, sequenceid=109, filesize=11.7 K 2024-12-09T08:14:47,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/4b4dcb65f0c94e9092ad07680dc4d42a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/4b4dcb65f0c94e9092ad07680dc4d42a 2024-12-09T08:14:47,832 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/4b4dcb65f0c94e9092ad07680dc4d42a, entries=150, sequenceid=109, filesize=11.7 K 2024-12-09T08:14:47,833 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c9202617e4fb01f766f444f5e419c460 in 1287ms, sequenceid=109, compaction requested=true 2024-12-09T08:14:47,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:47,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:47,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-09T08:14:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-09T08:14:47,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-09T08:14:47,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3690 sec 2024-12-09T08:14:47,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.3790 sec 2024-12-09T08:14:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:47,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T08:14:47,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:14:47,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:47,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:14:47,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:47,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:14:47,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:47,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099c2417b54bc54e7a8b421f82965852bf_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732086783/Put/seqid=0 2024-12-09T08:14:47,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732147948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732147951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732147953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732147953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732147956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:47,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741992_1168 (size=14594) 2024-12-09T08:14:48,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732148061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732148062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732148062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732148071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732148071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732148271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732148271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732148271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732148281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732148281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,371 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:48,374 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099c2417b54bc54e7a8b421f82965852bf_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099c2417b54bc54e7a8b421f82965852bf_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:48,375 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/508ab1dc41f94324ab09f41e62749e7b, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:48,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/508ab1dc41f94324ab09f41e62749e7b is 175, key is test_row_0/A:col10/1733732086783/Put/seqid=0 2024-12-09T08:14:48,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741993_1169 (size=39549) 2024-12-09T08:14:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732148581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732148581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732148582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732148591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732148591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:48,780 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/508ab1dc41f94324ab09f41e62749e7b 2024-12-09T08:14:48,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/d0a898b232ab4f4691c1ed594423d449 is 50, key is test_row_0/B:col10/1733732086783/Put/seqid=0 2024-12-09T08:14:48,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741994_1170 (size=12001) 2024-12-09T08:14:49,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732149091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:49,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732149091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:49,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732149091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:49,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732149101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732149101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:49,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/d0a898b232ab4f4691c1ed594423d449 2024-12-09T08:14:49,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/565e341df67b431e9fe2c4feaafba876 is 50, key is test_row_0/C:col10/1733732086783/Put/seqid=0 2024-12-09T08:14:49,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741995_1171 (size=12001) 2024-12-09T08:14:49,440 INFO [master/5bda946059d1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T08:14:49,440 INFO [master/5bda946059d1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T08:14:49,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/565e341df67b431e9fe2c4feaafba876 2024-12-09T08:14:49,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/508ab1dc41f94324ab09f41e62749e7b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/508ab1dc41f94324ab09f41e62749e7b 2024-12-09T08:14:49,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/508ab1dc41f94324ab09f41e62749e7b, entries=200, sequenceid=128, filesize=38.6 K 2024-12-09T08:14:49,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/d0a898b232ab4f4691c1ed594423d449 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/d0a898b232ab4f4691c1ed594423d449 2024-12-09T08:14:49,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/d0a898b232ab4f4691c1ed594423d449, entries=150, sequenceid=128, filesize=11.7 K 2024-12-09T08:14:49,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/565e341df67b431e9fe2c4feaafba876 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/565e341df67b431e9fe2c4feaafba876 2024-12-09T08:14:49,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/565e341df67b431e9fe2c4feaafba876, entries=150, sequenceid=128, filesize=11.7 K 2024-12-09T08:14:49,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for c9202617e4fb01f766f444f5e419c460 in 1689ms, sequenceid=128, compaction requested=true 2024-12-09T08:14:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:49,623 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:14:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:49,623 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:14:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:49,624 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:14:49,624 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:14:49,624 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/B is initiating minor compaction (all files) 2024-12-09T08:14:49,624 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/A is initiating minor compaction (all files) 2024-12-09T08:14:49,625 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/B in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:49,625 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/A in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:49,625 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/2662b2ae502e417380c272f410bc3fde, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/151e4c99ba044393a1450b52e0acaeab, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/18d61690743b4d218de0c5ae9e49204b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/d0a898b232ab4f4691c1ed594423d449] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=47.0 K 2024-12-09T08:14:49,625 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d2d5fc177a124a418267376765374b98, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e603db79e3a04427bfbcf4f787404c9b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/77c64fac696a412c953f3ad83018452a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/508ab1dc41f94324ab09f41e62749e7b] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=129.4 K 2024-12-09T08:14:49,625 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:49,625 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d2d5fc177a124a418267376765374b98, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e603db79e3a04427bfbcf4f787404c9b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/77c64fac696a412c953f3ad83018452a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/508ab1dc41f94324ab09f41e62749e7b] 2024-12-09T08:14:49,625 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2d5fc177a124a418267376765374b98, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733732084739 2024-12-09T08:14:49,625 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2662b2ae502e417380c272f410bc3fde, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733732084739 2024-12-09T08:14:49,625 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e603db79e3a04427bfbcf4f787404c9b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733732085571 2024-12-09T08:14:49,625 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 151e4c99ba044393a1450b52e0acaeab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733732085571 2024-12-09T08:14:49,626 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77c64fac696a412c953f3ad83018452a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733732085628 2024-12-09T08:14:49,626 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 18d61690743b4d218de0c5ae9e49204b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733732085628 2024-12-09T08:14:49,626 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 508ab1dc41f94324ab09f41e62749e7b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733732086783 2024-12-09T08:14:49,626 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a898b232ab4f4691c1ed594423d449, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733732086783 2024-12-09T08:14:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T08:14:49,631 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:49,632 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-09T08:14:49,634 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:49,635 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T08:14:49,636 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:49,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:49,644 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209e67f3a8e57ad40639bf1b62e7a231020_c9202617e4fb01f766f444f5e419c460 store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:49,646 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#B#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:49,646 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/2c8ae9b4cc7b4c429e55b1815785f521 is 50, key is test_row_0/B:col10/1733732086783/Put/seqid=0 2024-12-09T08:14:49,648 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209e67f3a8e57ad40639bf1b62e7a231020_c9202617e4fb01f766f444f5e419c460, store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:49,648 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e67f3a8e57ad40639bf1b62e7a231020_c9202617e4fb01f766f444f5e419c460 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:49,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741997_1173 (size=4469) 2024-12-09T08:14:49,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741996_1172 (size=12241) 2024-12-09T08:14:49,663 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/2c8ae9b4cc7b4c429e55b1815785f521 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/2c8ae9b4cc7b4c429e55b1815785f521 2024-12-09T08:14:49,669 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9202617e4fb01f766f444f5e419c460/B of c9202617e4fb01f766f444f5e419c460 into 2c8ae9b4cc7b4c429e55b1815785f521(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:49,669 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:49,669 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/B, priority=12, startTime=1733732089623; duration=0sec 2024-12-09T08:14:49,669 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:49,669 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:B 2024-12-09T08:14:49,669 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:14:49,671 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:14:49,671 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/C is initiating minor compaction (all files) 2024-12-09T08:14:49,671 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/C in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:49,671 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/887aa7927af44cba9b20348844a9c036, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/2b48ecde64af4cdb9cd6fb25a82bc9af, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/4b4dcb65f0c94e9092ad07680dc4d42a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/565e341df67b431e9fe2c4feaafba876] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=47.0 K 2024-12-09T08:14:49,672 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 887aa7927af44cba9b20348844a9c036, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733732084739 2024-12-09T08:14:49,672 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b48ecde64af4cdb9cd6fb25a82bc9af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733732085571 2024-12-09T08:14:49,673 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b4dcb65f0c94e9092ad07680dc4d42a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733732085628 2024-12-09T08:14:49,673 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 565e341df67b431e9fe2c4feaafba876, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733732086783 2024-12-09T08:14:49,684 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#C#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:49,684 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/599d8cec0d0346118401cb0dae52814f is 50, key is test_row_0/C:col10/1733732086783/Put/seqid=0 2024-12-09T08:14:49,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741998_1174 (size=12241) 2024-12-09T08:14:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T08:14:49,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T08:14:49,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:49,789 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T08:14:49,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:14:49,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:49,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:14:49,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:49,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:14:49,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:49,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f52caa0e21114aa9854c0e3283e6061a_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732087953/Put/seqid=0 2024-12-09T08:14:49,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741999_1175 (size=12304) 2024-12-09T08:14:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T08:14:50,058 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#A#compaction#149 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:50,059 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e7ea6d55204746f3ab394b1bdb9edf63 is 175, key is test_row_0/A:col10/1733732086783/Put/seqid=0 2024-12-09T08:14:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:50,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:50,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742000_1176 (size=31195) 2024-12-09T08:14:50,116 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/599d8cec0d0346118401cb0dae52814f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/599d8cec0d0346118401cb0dae52814f 2024-12-09T08:14:50,121 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9202617e4fb01f766f444f5e419c460/C of c9202617e4fb01f766f444f5e419c460 into 599d8cec0d0346118401cb0dae52814f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:50,121 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:50,121 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/C, priority=12, startTime=1733732089623; duration=0sec 2024-12-09T08:14:50,121 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:50,121 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:C 2024-12-09T08:14:50,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732150129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732150131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732150130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732150132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732150132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:50,205 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f52caa0e21114aa9854c0e3283e6061a_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f52caa0e21114aa9854c0e3283e6061a_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:50,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/8c2996c05a3f43788ca2ba2588090f45, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:50,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/8c2996c05a3f43788ca2ba2588090f45 is 175, key is test_row_0/A:col10/1733732087953/Put/seqid=0 2024-12-09T08:14:50,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742001_1177 (size=31105) 2024-12-09T08:14:50,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732150241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732150241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732150241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732150241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732150241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T08:14:50,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732150451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732150451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732150451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732150451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732150451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,521 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e7ea6d55204746f3ab394b1bdb9edf63 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e7ea6d55204746f3ab394b1bdb9edf63 2024-12-09T08:14:50,526 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9202617e4fb01f766f444f5e419c460/A of c9202617e4fb01f766f444f5e419c460 into e7ea6d55204746f3ab394b1bdb9edf63(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:50,526 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:50,526 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/A, priority=12, startTime=1733732089623; duration=0sec 2024-12-09T08:14:50,526 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:50,526 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:A 2024-12-09T08:14:50,620 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/8c2996c05a3f43788ca2ba2588090f45 2024-12-09T08:14:50,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/82f4d4ace4734a238f4c3cb7f141f9b1 is 50, key is test_row_0/B:col10/1733732087953/Put/seqid=0 2024-12-09T08:14:50,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742002_1178 (size=12151) 2024-12-09T08:14:50,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732150761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732150761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732150761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732150761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732150761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T08:14:51,032 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/82f4d4ace4734a238f4c3cb7f141f9b1 2024-12-09T08:14:51,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/b924fc7c619844eeb058930423cb25ab is 50, key is test_row_0/C:col10/1733732087953/Put/seqid=0 2024-12-09T08:14:51,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742003_1179 (size=12151) 2024-12-09T08:14:51,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732151271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:51,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732151271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732151271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732151271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732151272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:51,444 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/b924fc7c619844eeb058930423cb25ab 2024-12-09T08:14:51,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/8c2996c05a3f43788ca2ba2588090f45 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/8c2996c05a3f43788ca2ba2588090f45 2024-12-09T08:14:51,453 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/8c2996c05a3f43788ca2ba2588090f45, entries=150, sequenceid=143, filesize=30.4 K 2024-12-09T08:14:51,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/82f4d4ace4734a238f4c3cb7f141f9b1 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/82f4d4ace4734a238f4c3cb7f141f9b1 2024-12-09T08:14:51,458 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/82f4d4ace4734a238f4c3cb7f141f9b1, entries=150, sequenceid=143, filesize=11.9 K 2024-12-09T08:14:51,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/b924fc7c619844eeb058930423cb25ab as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/b924fc7c619844eeb058930423cb25ab 2024-12-09T08:14:51,463 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/b924fc7c619844eeb058930423cb25ab, entries=150, sequenceid=143, filesize=11.9 K 2024-12-09T08:14:51,464 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c9202617e4fb01f766f444f5e419c460 in 1675ms, sequenceid=143, compaction requested=false 2024-12-09T08:14:51,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:51,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:51,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-09T08:14:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-09T08:14:51,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-09T08:14:51,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8300 sec 2024-12-09T08:14:51,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8360 sec 2024-12-09T08:14:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T08:14:51,781 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:51,782 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-09T08:14:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T08:14:51,783 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:51,784 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:51,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T08:14:51,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T08:14:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:51,936 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T08:14:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:14:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:14:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:14:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:51,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ab52ddbb87434b52887dbae4a09b82a5_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732090131/Put/seqid=0 2024-12-09T08:14:51,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742004_1180 (size=12304) 2024-12-09T08:14:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T08:14:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:52,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:52,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732152292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732152292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732152294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732152299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732152300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:52,357 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ab52ddbb87434b52887dbae4a09b82a5_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab52ddbb87434b52887dbae4a09b82a5_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:52,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/27fe3069aeb74133a0762630bd399ba6, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:52,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/27fe3069aeb74133a0762630bd399ba6 is 175, key is test_row_0/A:col10/1733732090131/Put/seqid=0 2024-12-09T08:14:52,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742005_1181 (size=31105) 2024-12-09T08:14:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T08:14:52,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732152412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732152412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732152412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732152412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732152412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732152622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732152622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732152622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732152622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732152622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,763 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/27fe3069aeb74133a0762630bd399ba6 2024-12-09T08:14:52,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/d84510e0ccac4cfe87ecc10ac516b2f3 is 50, key is test_row_0/B:col10/1733732090131/Put/seqid=0 2024-12-09T08:14:52,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742006_1182 (size=12151) 2024-12-09T08:14:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T08:14:52,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732152931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732152931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732152931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732152931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732152932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,176 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/d84510e0ccac4cfe87ecc10ac516b2f3 2024-12-09T08:14:53,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/5807cc2b7e3441b880f20537cdb3f56b is 50, key is test_row_0/C:col10/1733732090131/Put/seqid=0 2024-12-09T08:14:53,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742007_1183 (size=12151) 2024-12-09T08:14:53,188 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/5807cc2b7e3441b880f20537cdb3f56b 2024-12-09T08:14:53,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/27fe3069aeb74133a0762630bd399ba6 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/27fe3069aeb74133a0762630bd399ba6 2024-12-09T08:14:53,198 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/27fe3069aeb74133a0762630bd399ba6, entries=150, sequenceid=163, filesize=30.4 K 2024-12-09T08:14:53,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/d84510e0ccac4cfe87ecc10ac516b2f3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/d84510e0ccac4cfe87ecc10ac516b2f3 2024-12-09T08:14:53,205 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/d84510e0ccac4cfe87ecc10ac516b2f3, entries=150, sequenceid=163, filesize=11.9 K 2024-12-09T08:14:53,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/5807cc2b7e3441b880f20537cdb3f56b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/5807cc2b7e3441b880f20537cdb3f56b 2024-12-09T08:14:53,211 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/5807cc2b7e3441b880f20537cdb3f56b, entries=150, sequenceid=163, filesize=11.9 K 2024-12-09T08:14:53,212 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c9202617e4fb01f766f444f5e419c460 in 1275ms, sequenceid=163, compaction requested=true 2024-12-09T08:14:53,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:53,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:53,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-09T08:14:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-09T08:14:53,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-09T08:14:53,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4290 sec 2024-12-09T08:14:53,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4340 sec 2024-12-09T08:14:53,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:53,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T08:14:53,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:14:53,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:53,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:14:53,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:53,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:14:53,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:53,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c5907ad65e6f4d43a61d18fc5e197a63_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732093443/Put/seqid=0 2024-12-09T08:14:53,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732153460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732153461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732153461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732153462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732153462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742008_1184 (size=19774) 2024-12-09T08:14:53,474 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:53,478 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c5907ad65e6f4d43a61d18fc5e197a63_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c5907ad65e6f4d43a61d18fc5e197a63_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:53,479 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/d0f9a9744660484b8cd14088cf55dc18, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:53,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/d0f9a9744660484b8cd14088cf55dc18 is 175, key is test_row_0/A:col10/1733732093443/Put/seqid=0 2024-12-09T08:14:53,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742009_1185 (size=57033) 2024-12-09T08:14:53,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732153571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732153571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732153571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732153571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732153572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732153781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732153781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732153781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732153782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732153782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:53,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:53,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:53,884 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/d0f9a9744660484b8cd14088cf55dc18 2024-12-09T08:14:53,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/25bee722fb954b36adbed4ef4750202c is 50, key is test_row_0/B:col10/1733732093443/Put/seqid=0 2024-12-09T08:14:53,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742010_1186 (size=12151) 2024-12-09T08:14:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T08:14:53,931 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:53,932 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-09T08:14:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T08:14:53,934 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:53,934 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:53,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T08:14:54,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T08:14:54,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:54,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:54,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732154091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732154091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:54,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:54,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732154091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732154091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732154091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:54,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T08:14:54,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:54,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T08:14:54,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/25bee722fb954b36adbed4ef4750202c 2024-12-09T08:14:54,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/e34ebbba083f4620972f8b0031981b00 is 50, key is test_row_0/C:col10/1733732093443/Put/seqid=0 2024-12-09T08:14:54,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742011_1187 (size=12151) 2024-12-09T08:14:54,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T08:14:54,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:54,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T08:14:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T08:14:54,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732154601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:54,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732154611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732154611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732154611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:54,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732154611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:54,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T08:14:54,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:54,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:54,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/e34ebbba083f4620972f8b0031981b00 2024-12-09T08:14:54,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/d0f9a9744660484b8cd14088cf55dc18 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d0f9a9744660484b8cd14088cf55dc18 2024-12-09T08:14:54,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d0f9a9744660484b8cd14088cf55dc18, entries=300, sequenceid=180, filesize=55.7 K 2024-12-09T08:14:54,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/25bee722fb954b36adbed4ef4750202c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/25bee722fb954b36adbed4ef4750202c 2024-12-09T08:14:54,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/25bee722fb954b36adbed4ef4750202c, entries=150, sequenceid=180, filesize=11.9 K 2024-12-09T08:14:54,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/e34ebbba083f4620972f8b0031981b00 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/e34ebbba083f4620972f8b0031981b00 2024-12-09T08:14:54,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/e34ebbba083f4620972f8b0031981b00, entries=150, sequenceid=180, filesize=11.9 K 2024-12-09T08:14:54,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for c9202617e4fb01f766f444f5e419c460 in 1301ms, sequenceid=180, compaction requested=true 2024-12-09T08:14:54,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:54,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:54,744 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:14:54,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:54,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:54,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:54,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:54,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T08:14:54,745 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:14:54,746 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:14:54,746 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:14:54,746 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/B is initiating minor compaction (all files) 2024-12-09T08:14:54,746 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/A is initiating minor compaction (all files) 2024-12-09T08:14:54,746 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/B in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,746 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/A in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,746 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e7ea6d55204746f3ab394b1bdb9edf63, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/8c2996c05a3f43788ca2ba2588090f45, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/27fe3069aeb74133a0762630bd399ba6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d0f9a9744660484b8cd14088cf55dc18] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=146.9 K 2024-12-09T08:14:54,746 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/2c8ae9b4cc7b4c429e55b1815785f521, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/82f4d4ace4734a238f4c3cb7f141f9b1, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/d84510e0ccac4cfe87ecc10ac516b2f3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/25bee722fb954b36adbed4ef4750202c] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=47.6 K 2024-12-09T08:14:54,746 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,746 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e7ea6d55204746f3ab394b1bdb9edf63, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/8c2996c05a3f43788ca2ba2588090f45, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/27fe3069aeb74133a0762630bd399ba6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d0f9a9744660484b8cd14088cf55dc18] 2024-12-09T08:14:54,746 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c8ae9b4cc7b4c429e55b1815785f521, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733732086783 2024-12-09T08:14:54,747 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7ea6d55204746f3ab394b1bdb9edf63, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733732086783 2024-12-09T08:14:54,747 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 82f4d4ace4734a238f4c3cb7f141f9b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733732087945 2024-12-09T08:14:54,747 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c2996c05a3f43788ca2ba2588090f45, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733732087945 2024-12-09T08:14:54,747 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting d84510e0ccac4cfe87ecc10ac516b2f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733732090130 2024-12-09T08:14:54,747 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27fe3069aeb74133a0762630bd399ba6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733732090130 2024-12-09T08:14:54,747 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0f9a9744660484b8cd14088cf55dc18, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733732092286 2024-12-09T08:14:54,748 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 25bee722fb954b36adbed4ef4750202c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733732092299 2024-12-09T08:14:54,757 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:54,758 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#B#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:54,758 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/5e1bc0e89f2e412aa5ae47bb6faef1fb is 50, key is test_row_0/B:col10/1733732093443/Put/seqid=0 2024-12-09T08:14:54,768 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412092f6b79a7cc97475d97b0467666538d2e_c9202617e4fb01f766f444f5e419c460 store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:54,769 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412092f6b79a7cc97475d97b0467666538d2e_c9202617e4fb01f766f444f5e419c460, store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:54,770 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092f6b79a7cc97475d97b0467666538d2e_c9202617e4fb01f766f444f5e419c460 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:54,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742012_1188 (size=12527) 2024-12-09T08:14:54,779 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/5e1bc0e89f2e412aa5ae47bb6faef1fb as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/5e1bc0e89f2e412aa5ae47bb6faef1fb 2024-12-09T08:14:54,787 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9202617e4fb01f766f444f5e419c460/B of c9202617e4fb01f766f444f5e419c460 into 5e1bc0e89f2e412aa5ae47bb6faef1fb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:54,787 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:54,787 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/B, priority=12, startTime=1733732094744; duration=0sec 2024-12-09T08:14:54,787 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:54,787 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:B 2024-12-09T08:14:54,787 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:14:54,789 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:14:54,789 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/C is initiating minor compaction (all files) 2024-12-09T08:14:54,789 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/C in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,789 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/599d8cec0d0346118401cb0dae52814f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/b924fc7c619844eeb058930423cb25ab, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/5807cc2b7e3441b880f20537cdb3f56b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/e34ebbba083f4620972f8b0031981b00] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=47.6 K 2024-12-09T08:14:54,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742013_1189 (size=4469) 2024-12-09T08:14:54,790 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 599d8cec0d0346118401cb0dae52814f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733732086783 2024-12-09T08:14:54,791 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting b924fc7c619844eeb058930423cb25ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733732087945 2024-12-09T08:14:54,791 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#A#compaction#162 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:54,791 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/32a79536f6484b34b202a04a1701ff5f is 175, key is test_row_0/A:col10/1733732093443/Put/seqid=0 2024-12-09T08:14:54,792 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5807cc2b7e3441b880f20537cdb3f56b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733732090130 2024-12-09T08:14:54,793 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting e34ebbba083f4620972f8b0031981b00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733732092299 2024-12-09T08:14:54,811 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#C#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:54,812 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/ac9de0dc15314befb77569348f8400dd is 50, key is test_row_0/C:col10/1733732093443/Put/seqid=0 2024-12-09T08:14:54,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742014_1190 (size=31481) 2024-12-09T08:14:54,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742015_1191 (size=12527) 2024-12-09T08:14:54,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T08:14:54,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:54,850 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T08:14:54,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:14:54,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:54,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:14:54,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:54,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:14:54,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:54,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120988e8578e468c45f29ca7197b3a3293f4_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732093458/Put/seqid=0 2024-12-09T08:14:54,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742016_1192 (size=12304) 2024-12-09T08:14:54,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:54,874 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120988e8578e468c45f29ca7197b3a3293f4_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120988e8578e468c45f29ca7197b3a3293f4_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:54,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/0a0f1ae1dde44aa38c04cf325bce2ccd, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:54,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/0a0f1ae1dde44aa38c04cf325bce2ccd is 175, key is test_row_0/A:col10/1733732093458/Put/seqid=0 2024-12-09T08:14:54,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742017_1193 (size=31105) 2024-12-09T08:14:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T08:14:55,253 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/ac9de0dc15314befb77569348f8400dd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/ac9de0dc15314befb77569348f8400dd 2024-12-09T08:14:55,253 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/32a79536f6484b34b202a04a1701ff5f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/32a79536f6484b34b202a04a1701ff5f 2024-12-09T08:14:55,259 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9202617e4fb01f766f444f5e419c460/A of c9202617e4fb01f766f444f5e419c460 into 32a79536f6484b34b202a04a1701ff5f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:55,259 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9202617e4fb01f766f444f5e419c460/C of c9202617e4fb01f766f444f5e419c460 into ac9de0dc15314befb77569348f8400dd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:55,259 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:55,259 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:55,259 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/C, priority=12, startTime=1733732094744; duration=0sec 2024-12-09T08:14:55,259 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/A, priority=12, startTime=1733732094744; duration=0sec 2024-12-09T08:14:55,259 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:55,259 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:55,259 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:C 2024-12-09T08:14:55,259 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:A 2024-12-09T08:14:55,285 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/0a0f1ae1dde44aa38c04cf325bce2ccd 2024-12-09T08:14:55,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/6f6afc8706e341e5b0c9c85a7d6919e2 is 50, key is test_row_0/B:col10/1733732093458/Put/seqid=0 2024-12-09T08:14:55,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742018_1194 (size=12151) 2024-12-09T08:14:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:55,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:55,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732155628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732155631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732155632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732155634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732155635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,699 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/6f6afc8706e341e5b0c9c85a7d6919e2 2024-12-09T08:14:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/279c63ec956e43649379a4479efd0356 is 50, key is test_row_0/C:col10/1733732093458/Put/seqid=0 2024-12-09T08:14:55,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742019_1195 (size=12151) 2024-12-09T08:14:55,727 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/279c63ec956e43649379a4479efd0356 2024-12-09T08:14:55,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/0a0f1ae1dde44aa38c04cf325bce2ccd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/0a0f1ae1dde44aa38c04cf325bce2ccd 2024-12-09T08:14:55,735 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/0a0f1ae1dde44aa38c04cf325bce2ccd, entries=150, sequenceid=197, filesize=30.4 K 2024-12-09T08:14:55,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/6f6afc8706e341e5b0c9c85a7d6919e2 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/6f6afc8706e341e5b0c9c85a7d6919e2 2024-12-09T08:14:55,741 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/6f6afc8706e341e5b0c9c85a7d6919e2, entries=150, sequenceid=197, filesize=11.9 K 2024-12-09T08:14:55,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732155741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732155742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732155742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732155742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/279c63ec956e43649379a4479efd0356 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/279c63ec956e43649379a4479efd0356 2024-12-09T08:14:55,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732155742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,749 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/279c63ec956e43649379a4479efd0356, entries=150, sequenceid=197, filesize=11.9 K 2024-12-09T08:14:55,750 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c9202617e4fb01f766f444f5e419c460 in 900ms, sequenceid=197, compaction requested=false 2024-12-09T08:14:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-09T08:14:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-09T08:14:55,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-09T08:14:55,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8170 sec 2024-12-09T08:14:55,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8220 sec 2024-12-09T08:14:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:55,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T08:14:55,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:14:55,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:55,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:14:55,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:55,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:14:55,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:55,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099b4b65cc0043462c865156be96131851_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732095632/Put/seqid=0 2024-12-09T08:14:55,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732155965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732155966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742020_1196 (size=17284) 2024-12-09T08:14:55,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732155968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732155969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732155969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732156071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732156081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732156081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732156081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732156081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T08:14:56,091 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:14:56,091 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:14:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-09T08:14:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T08:14:56,094 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:14:56,094 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:14:56,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:14:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T08:14:56,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T08:14:56,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:56,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:56,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:56,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732156281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732156291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732156291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732156291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732156291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,372 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:56,378 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099b4b65cc0043462c865156be96131851_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099b4b65cc0043462c865156be96131851_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:56,380 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e084d878a6c14aaa9cf14ad07eab9113, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:56,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e084d878a6c14aaa9cf14ad07eab9113 is 175, key is test_row_0/A:col10/1733732095632/Put/seqid=0 2024-12-09T08:14:56,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742021_1197 (size=48389) 2024-12-09T08:14:56,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T08:14:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:56,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T08:14:56,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T08:14:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:56,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732156591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732156601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732156601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732156608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732156608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:56,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T08:14:56,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:56,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:56,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:56,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T08:14:56,787 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e084d878a6c14aaa9cf14ad07eab9113 2024-12-09T08:14:56,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/4a16aa6091764e2796c8b124906f1f50 is 50, key is test_row_0/B:col10/1733732095632/Put/seqid=0 2024-12-09T08:14:56,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742022_1198 (size=12151) 2024-12-09T08:14:56,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T08:14:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:56,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T08:14:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732157101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:57,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732157111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:57,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732157111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:57,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732157121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:57,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732157121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:57,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T08:14:57,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:57,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/4a16aa6091764e2796c8b124906f1f50 2024-12-09T08:14:57,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/05a0dcfacc004a208ab31af99c29a922 is 50, key is test_row_0/C:col10/1733732095632/Put/seqid=0 2024-12-09T08:14:57,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742023_1199 (size=12151) 2024-12-09T08:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T08:14:57,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T08:14:57,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:57,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T08:14:57,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:57,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/05a0dcfacc004a208ab31af99c29a922 2024-12-09T08:14:57,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e084d878a6c14aaa9cf14ad07eab9113 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e084d878a6c14aaa9cf14ad07eab9113 2024-12-09T08:14:57,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T08:14:57,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:57,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:14:57,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e084d878a6c14aaa9cf14ad07eab9113, entries=250, sequenceid=217, filesize=47.3 K 2024-12-09T08:14:57,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/4a16aa6091764e2796c8b124906f1f50 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/4a16aa6091764e2796c8b124906f1f50 2024-12-09T08:14:57,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/4a16aa6091764e2796c8b124906f1f50, entries=150, sequenceid=217, filesize=11.9 K 2024-12-09T08:14:57,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/05a0dcfacc004a208ab31af99c29a922 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/05a0dcfacc004a208ab31af99c29a922 2024-12-09T08:14:57,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/05a0dcfacc004a208ab31af99c29a922, entries=150, sequenceid=217, filesize=11.9 K 2024-12-09T08:14:57,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c9202617e4fb01f766f444f5e419c460 in 1685ms, sequenceid=217, compaction requested=true 2024-12-09T08:14:57,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:57,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:14:57,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:57,638 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:57,638 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:57,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:14:57,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:57,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:14:57,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:57,639 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:57,639 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:57,639 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/B is initiating minor compaction (all files) 2024-12-09T08:14:57,639 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/A is initiating minor compaction (all files) 2024-12-09T08:14:57,639 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/B in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,639 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/A in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,639 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/5e1bc0e89f2e412aa5ae47bb6faef1fb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/6f6afc8706e341e5b0c9c85a7d6919e2, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/4a16aa6091764e2796c8b124906f1f50] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=36.0 K 2024-12-09T08:14:57,639 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/32a79536f6484b34b202a04a1701ff5f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/0a0f1ae1dde44aa38c04cf325bce2ccd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e084d878a6c14aaa9cf14ad07eab9113] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=108.4 K 2024-12-09T08:14:57,639 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,639 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/32a79536f6484b34b202a04a1701ff5f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/0a0f1ae1dde44aa38c04cf325bce2ccd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e084d878a6c14aaa9cf14ad07eab9113] 2024-12-09T08:14:57,639 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e1bc0e89f2e412aa5ae47bb6faef1fb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733732092299 2024-12-09T08:14:57,640 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32a79536f6484b34b202a04a1701ff5f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733732092299 2024-12-09T08:14:57,640 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f6afc8706e341e5b0c9c85a7d6919e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733732093458 2024-12-09T08:14:57,640 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a0f1ae1dde44aa38c04cf325bce2ccd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733732093458 2024-12-09T08:14:57,640 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a16aa6091764e2796c8b124906f1f50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733732095632 2024-12-09T08:14:57,640 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e084d878a6c14aaa9cf14ad07eab9113, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733732095629 2024-12-09T08:14:57,649 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:57,649 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#B#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:57,650 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/c77a9960b5eb4369a040d4cfbbc928a5 is 50, key is test_row_0/B:col10/1733732095632/Put/seqid=0 2024-12-09T08:14:57,658 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209e19cc4b5c75449f1a8aade450b7e56b2_c9202617e4fb01f766f444f5e419c460 store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:57,660 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209e19cc4b5c75449f1a8aade450b7e56b2_c9202617e4fb01f766f444f5e419c460, store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:57,660 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e19cc4b5c75449f1a8aade450b7e56b2_c9202617e4fb01f766f444f5e419c460 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:57,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742024_1200 (size=12629) 2024-12-09T08:14:57,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742025_1201 (size=4469) 2024-12-09T08:14:57,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T08:14:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:57,779 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T08:14:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:14:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:14:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:14:57,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:14:57,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099dc4d2ba481b4769807e3e77405cb71f_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732095965/Put/seqid=0 2024-12-09T08:14:57,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742026_1202 (size=12304) 2024-12-09T08:14:58,077 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#A#compaction#171 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:58,078 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/093304ec3c854b7db3cc958c35447a4a is 175, key is test_row_0/A:col10/1733732095632/Put/seqid=0 2024-12-09T08:14:58,081 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/c77a9960b5eb4369a040d4cfbbc928a5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/c77a9960b5eb4369a040d4cfbbc928a5 2024-12-09T08:14:58,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742027_1203 (size=31583) 2024-12-09T08:14:58,087 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/093304ec3c854b7db3cc958c35447a4a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/093304ec3c854b7db3cc958c35447a4a 2024-12-09T08:14:58,090 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9202617e4fb01f766f444f5e419c460/B of c9202617e4fb01f766f444f5e419c460 into c77a9960b5eb4369a040d4cfbbc928a5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:58,090 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:58,091 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/B, priority=13, startTime=1733732097638; duration=0sec 2024-12-09T08:14:58,091 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:14:58,091 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:B 2024-12-09T08:14:58,091 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:14:58,092 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:14:58,092 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/C is initiating minor compaction (all files) 2024-12-09T08:14:58,092 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/C in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:58,092 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/ac9de0dc15314befb77569348f8400dd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/279c63ec956e43649379a4479efd0356, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/05a0dcfacc004a208ab31af99c29a922] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=36.0 K 2024-12-09T08:14:58,092 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9202617e4fb01f766f444f5e419c460/A of c9202617e4fb01f766f444f5e419c460 into 093304ec3c854b7db3cc958c35447a4a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:58,093 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ac9de0dc15314befb77569348f8400dd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733732092299 2024-12-09T08:14:58,093 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:58,093 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/A, priority=13, startTime=1733732097638; duration=0sec 2024-12-09T08:14:58,093 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:58,093 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:A 2024-12-09T08:14:58,093 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 279c63ec956e43649379a4479efd0356, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733732093458 2024-12-09T08:14:58,093 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 05a0dcfacc004a208ab31af99c29a922, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733732095632 2024-12-09T08:14:58,099 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#C#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:14:58,100 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/9ed5325afd784a49b4591d68ddc81b58 is 50, key is test_row_0/C:col10/1733732095632/Put/seqid=0 2024-12-09T08:14:58,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742028_1204 (size=12629) 2024-12-09T08:14:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:58,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:14:58,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732158134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732158136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732158136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732158137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732158138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:14:58,194 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099dc4d2ba481b4769807e3e77405cb71f_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099dc4d2ba481b4769807e3e77405cb71f_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:14:58,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/f2555a2f23334c04860c993512de1135, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:14:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/f2555a2f23334c04860c993512de1135 is 175, key is test_row_0/A:col10/1733732095965/Put/seqid=0 2024-12-09T08:14:58,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742029_1205 (size=31105) 2024-12-09T08:14:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T08:14:58,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732158241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732158242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732158242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732158251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732158251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732158451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732158450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732158451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732158461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732158461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,510 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/9ed5325afd784a49b4591d68ddc81b58 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/9ed5325afd784a49b4591d68ddc81b58 2024-12-09T08:14:58,515 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9202617e4fb01f766f444f5e419c460/C of c9202617e4fb01f766f444f5e419c460 into 9ed5325afd784a49b4591d68ddc81b58(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:14:58,515 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:58,515 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/C, priority=13, startTime=1733732097638; duration=0sec 2024-12-09T08:14:58,515 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:14:58,515 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:C 2024-12-09T08:14:58,600 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/f2555a2f23334c04860c993512de1135 2024-12-09T08:14:58,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/62f81c854b0f473a95b052b2230b7814 is 50, key is test_row_0/B:col10/1733732095965/Put/seqid=0 2024-12-09T08:14:58,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742030_1206 (size=12151) 2024-12-09T08:14:58,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732158761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732158760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732158761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732158771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:58,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732158771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:59,010 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/62f81c854b0f473a95b052b2230b7814 2024-12-09T08:14:59,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/46c16d78306942e19c43c5b10d1fefd8 is 50, key is test_row_0/C:col10/1733732095965/Put/seqid=0 2024-12-09T08:14:59,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742031_1207 (size=12151) 2024-12-09T08:14:59,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732159271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:59,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732159271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:59,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732159271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:59,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732159281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:59,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:59,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:14:59,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:59,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732159281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:14:59,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:59,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:14:59,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:14:59,430 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/46c16d78306942e19c43c5b10d1fefd8 2024-12-09T08:14:59,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/f2555a2f23334c04860c993512de1135 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f2555a2f23334c04860c993512de1135 2024-12-09T08:14:59,438 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f2555a2f23334c04860c993512de1135, entries=150, sequenceid=231, filesize=30.4 K 2024-12-09T08:14:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/62f81c854b0f473a95b052b2230b7814 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/62f81c854b0f473a95b052b2230b7814 2024-12-09T08:14:59,443 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/62f81c854b0f473a95b052b2230b7814, entries=150, sequenceid=231, filesize=11.9 K 2024-12-09T08:14:59,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/46c16d78306942e19c43c5b10d1fefd8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/46c16d78306942e19c43c5b10d1fefd8 2024-12-09T08:14:59,447 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/46c16d78306942e19c43c5b10d1fefd8, entries=150, sequenceid=231, filesize=11.9 K 2024-12-09T08:14:59,449 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c9202617e4fb01f766f444f5e419c460 in 1670ms, sequenceid=231, compaction requested=false 2024-12-09T08:14:59,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:14:59,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:14:59,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-09T08:14:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-09T08:14:59,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-09T08:14:59,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3560 sec 2024-12-09T08:14:59,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 3.3610 sec 2024-12-09T08:15:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T08:15:00,252 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:00,252 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-09T08:15:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T08:15:00,253 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:00,254 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:00,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:00,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T08:15:00,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:15:00,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:00,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:15:00,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:00,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:15:00,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:00,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732160294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732160295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732160296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732160296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732160301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,304 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in c9202617e4fb01f766f444f5e419c460 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:00,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209888fbf9d97304abeb4f8d44eb80d31fc_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732100283/Put/seqid=0 2024-12-09T08:15:00,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742032_1208 (size=17284) 2024-12-09T08:15:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T08:15:00,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732160401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T08:15:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732160401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:00,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:15:00,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:00,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732160401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:00,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:00,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732160410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T08:15:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:15:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:00,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T08:15:00,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732160611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732160611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732160611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732160621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T08:15:00,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:00,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:15:00,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:00,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:00,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:00,721 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:00,725 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209888fbf9d97304abeb4f8d44eb80d31fc_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209888fbf9d97304abeb4f8d44eb80d31fc_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:00,726 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/c269b84b6e53480c94fecc63cf71333b, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:00,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/c269b84b6e53480c94fecc63cf71333b is 175, key is test_row_0/A:col10/1733732100283/Put/seqid=0 2024-12-09T08:15:00,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742033_1209 (size=48389) 2024-12-09T08:15:00,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T08:15:00,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:00,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:15:00,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:00,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:00,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:00,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T08:15:00,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732160921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732160921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732160921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:00,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732160931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:01,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T08:15:01,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:01,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:15:01,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:01,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:01,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:01,131 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/c269b84b6e53480c94fecc63cf71333b 2024-12-09T08:15:01,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/e92d7ac8eed541b39060d585dfb9475b is 50, key is test_row_0/B:col10/1733732100283/Put/seqid=0 2024-12-09T08:15:01,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742034_1210 (size=12151) 2024-12-09T08:15:01,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T08:15:01,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:01,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:15:01,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:01,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:01,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:01,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T08:15:01,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:01,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:15:01,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:01,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:01,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T08:15:01,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732161431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:01,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732161431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:01,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732161431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732161441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:01,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T08:15:01,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:01,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:15:01,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:01,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:01,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:01,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/e92d7ac8eed541b39060d585dfb9475b 2024-12-09T08:15:01,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/1cd49ce8b3d0469ba6fca90b27eae989 is 50, key is test_row_0/C:col10/1733732100283/Put/seqid=0 2024-12-09T08:15:01,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742035_1211 (size=12151) 2024-12-09T08:15:01,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/1cd49ce8b3d0469ba6fca90b27eae989 2024-12-09T08:15:01,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/c269b84b6e53480c94fecc63cf71333b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/c269b84b6e53480c94fecc63cf71333b 2024-12-09T08:15:01,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/c269b84b6e53480c94fecc63cf71333b, entries=250, sequenceid=255, filesize=47.3 K 2024-12-09T08:15:01,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/e92d7ac8eed541b39060d585dfb9475b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/e92d7ac8eed541b39060d585dfb9475b 2024-12-09T08:15:01,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/e92d7ac8eed541b39060d585dfb9475b, entries=150, sequenceid=255, filesize=11.9 K 2024-12-09T08:15:01,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/1cd49ce8b3d0469ba6fca90b27eae989 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/1cd49ce8b3d0469ba6fca90b27eae989 2024-12-09T08:15:01,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/1cd49ce8b3d0469ba6fca90b27eae989, entries=150, sequenceid=255, filesize=11.9 K 2024-12-09T08:15:01,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c9202617e4fb01f766f444f5e419c460 in 1293ms, sequenceid=255, compaction requested=true 2024-12-09T08:15:01,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:15:01,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:01,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:01,577 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:01,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:01,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:01,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:01,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T08:15:01,579 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:01,580 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111077 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:01,581 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/A is initiating minor compaction (all files) 2024-12-09T08:15:01,581 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/A in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:01,581 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/093304ec3c854b7db3cc958c35447a4a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f2555a2f23334c04860c993512de1135, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/c269b84b6e53480c94fecc63cf71333b] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=108.5 K 2024-12-09T08:15:01,581 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:01,581 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/093304ec3c854b7db3cc958c35447a4a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f2555a2f23334c04860c993512de1135, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/c269b84b6e53480c94fecc63cf71333b] 2024-12-09T08:15:01,582 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:01,582 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/B is initiating minor compaction (all files) 2024-12-09T08:15:01,582 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/B in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:01,582 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/c77a9960b5eb4369a040d4cfbbc928a5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/62f81c854b0f473a95b052b2230b7814, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/e92d7ac8eed541b39060d585dfb9475b] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=36.1 K 2024-12-09T08:15:01,582 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 093304ec3c854b7db3cc958c35447a4a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733732095632 2024-12-09T08:15:01,583 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting c77a9960b5eb4369a040d4cfbbc928a5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733732095632 2024-12-09T08:15:01,583 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2555a2f23334c04860c993512de1135, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733732095964 2024-12-09T08:15:01,583 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c269b84b6e53480c94fecc63cf71333b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733732098135 2024-12-09T08:15:01,584 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f81c854b0f473a95b052b2230b7814, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733732095964 2024-12-09T08:15:01,584 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting e92d7ac8eed541b39060d585dfb9475b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733732100283 2024-12-09T08:15:01,589 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:01,594 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209e8b1702da32848c2bc1772a532fd121b_c9202617e4fb01f766f444f5e419c460 store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:01,595 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#B#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:01,596 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/dc7528745d52491a850dba594b4a0369 is 50, key is test_row_0/B:col10/1733732100283/Put/seqid=0 2024-12-09T08:15:01,596 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209e8b1702da32848c2bc1772a532fd121b_c9202617e4fb01f766f444f5e419c460, store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:01,596 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e8b1702da32848c2bc1772a532fd121b_c9202617e4fb01f766f444f5e419c460 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:01,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742037_1213 (size=4469) 2024-12-09T08:15:01,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742036_1212 (size=12731) 2024-12-09T08:15:01,601 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#A#compaction#179 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:01,601 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/71aeb6de437840629b57c2ce86a4c660 is 175, key is test_row_0/A:col10/1733732100283/Put/seqid=0 2024-12-09T08:15:01,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742038_1214 (size=31685) 2024-12-09T08:15:01,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T08:15:01,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:01,628 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:15:01,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:15:01,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:01,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:15:01,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:01,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:15:01,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:01,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209cf1e6c18dbba425d8241439e7bb373ca_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732100291/Put/seqid=0 2024-12-09T08:15:01,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742039_1215 (size=12454) 2024-12-09T08:15:02,004 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/dc7528745d52491a850dba594b4a0369 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/dc7528745d52491a850dba594b4a0369 2024-12-09T08:15:02,009 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/71aeb6de437840629b57c2ce86a4c660 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/71aeb6de437840629b57c2ce86a4c660 2024-12-09T08:15:02,010 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9202617e4fb01f766f444f5e419c460/B of c9202617e4fb01f766f444f5e419c460 into dc7528745d52491a850dba594b4a0369(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:02,010 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:15:02,010 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/B, priority=13, startTime=1733732101577; duration=0sec 2024-12-09T08:15:02,010 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:02,010 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:B 2024-12-09T08:15:02,010 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:02,011 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:02,011 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/C is initiating minor compaction (all files) 2024-12-09T08:15:02,011 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/C in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:02,012 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/9ed5325afd784a49b4591d68ddc81b58, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/46c16d78306942e19c43c5b10d1fefd8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/1cd49ce8b3d0469ba6fca90b27eae989] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=36.1 K 2024-12-09T08:15:02,012 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ed5325afd784a49b4591d68ddc81b58, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733732095632 2024-12-09T08:15:02,012 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 46c16d78306942e19c43c5b10d1fefd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733732095964 2024-12-09T08:15:02,013 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cd49ce8b3d0469ba6fca90b27eae989, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733732100283 2024-12-09T08:15:02,014 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9202617e4fb01f766f444f5e419c460/A of c9202617e4fb01f766f444f5e419c460 into 71aeb6de437840629b57c2ce86a4c660(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:02,014 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:15:02,014 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/A, priority=13, startTime=1733732101577; duration=0sec 2024-12-09T08:15:02,015 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:02,015 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:A 2024-12-09T08:15:02,019 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#C#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:02,020 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/677dfc1c1ab141e381e57653c0c5c7f3 is 50, key is test_row_0/C:col10/1733732100283/Put/seqid=0 2024-12-09T08:15:02,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742040_1216 (size=12731) 2024-12-09T08:15:02,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:02,042 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209cf1e6c18dbba425d8241439e7bb373ca_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209cf1e6c18dbba425d8241439e7bb373ca_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:02,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e44e7d54ba574e64ba15927ad99fc33f, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:02,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e44e7d54ba574e64ba15927ad99fc33f is 175, key is test_row_0/A:col10/1733732100291/Put/seqid=0 2024-12-09T08:15:02,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742041_1217 (size=31255) 2024-12-09T08:15:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:02,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. as already flushing 2024-12-09T08:15:02,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732102336, max timestamp=1733732102357Num uniques 859; 2024-12-09T08:15:02,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732102336, max timestamp=1733732102357Num uniques 859; 2024-12-09T08:15:02,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732102336, max timestamp=1733732102357Num uniques 859; 2024-12-09T08:15:02,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:02,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:02,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T08:15:02,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732162407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:02,434 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/677dfc1c1ab141e381e57653c0c5c7f3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/677dfc1c1ab141e381e57653c0c5c7f3 2024-12-09T08:15:02,438 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9202617e4fb01f766f444f5e419c460/C of c9202617e4fb01f766f444f5e419c460 into 677dfc1c1ab141e381e57653c0c5c7f3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:02,438 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:15:02,438 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/C, priority=13, startTime=1733732101578; duration=0sec 2024-12-09T08:15:02,439 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:02,439 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:C 2024-12-09T08:15:02,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:02,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41766 deadline: 1733732162441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41788 deadline: 1733732162441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:02,445 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in c9202617e4fb01f766f444f5e419c460 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:02,445 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in c9202617e4fb01f766f444f5e419c460 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:02,448 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e44e7d54ba574e64ba15927ad99fc33f 2024-12-09T08:15:02,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:02,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732162451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/0995f7c8cd1b4e4a9baadbbae665eef4 is 50, key is test_row_0/B:col10/1733732100291/Put/seqid=0 2024-12-09T08:15:02,455 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in c9202617e4fb01f766f444f5e419c460 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:02,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742042_1218 (size=12301) 2024-12-09T08:15:02,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:02,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732162461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:02,465 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/0995f7c8cd1b4e4a9baadbbae665eef4 2024-12-09T08:15:02,465 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in c9202617e4fb01f766f444f5e419c460 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:02,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/fa63c20267544db68818cd747a1194a3 is 50, key is test_row_0/C:col10/1733732100291/Put/seqid=0 2024-12-09T08:15:02,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742043_1219 (size=12301) 2024-12-09T08:15:02,511 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T08:15:02,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:02,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732162521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:02,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732162731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:02,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:02,875 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/fa63c20267544db68818cd747a1194a3 2024-12-09T08:15:02,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/e44e7d54ba574e64ba15927ad99fc33f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e44e7d54ba574e64ba15927ad99fc33f 2024-12-09T08:15:02,882 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e44e7d54ba574e64ba15927ad99fc33f, entries=150, sequenceid=266, filesize=30.5 K 2024-12-09T08:15:02,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/0995f7c8cd1b4e4a9baadbbae665eef4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/0995f7c8cd1b4e4a9baadbbae665eef4 2024-12-09T08:15:02,887 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/0995f7c8cd1b4e4a9baadbbae665eef4, entries=150, sequenceid=266, filesize=12.0 K 2024-12-09T08:15:02,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/fa63c20267544db68818cd747a1194a3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/fa63c20267544db68818cd747a1194a3 2024-12-09T08:15:02,894 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/fa63c20267544db68818cd747a1194a3, entries=150, sequenceid=266, filesize=12.0 K 2024-12-09T08:15:02,895 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c9202617e4fb01f766f444f5e419c460 in 1267ms, sequenceid=266, compaction requested=false 2024-12-09T08:15:02,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:15:02,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:02,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-09T08:15:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-09T08:15:02,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-09T08:15:02,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6420 sec 2024-12-09T08:15:02,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.6470 sec 2024-12-09T08:15:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:03,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T08:15:03,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:15:03,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:03,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:15:03,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:03,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:15:03,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:03,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f790f468c25f4fec98f69c46b1472a6d_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732103043/Put/seqid=0 2024-12-09T08:15:03,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742044_1220 (size=14994) 2024-12-09T08:15:03,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732163060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:03,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:03,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:03,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732163171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:03,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732163381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:03,454 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:03,459 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f790f468c25f4fec98f69c46b1472a6d_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f790f468c25f4fec98f69c46b1472a6d_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:03,459 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/5d46c13437054da497e254fe9f447d55, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:03,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/5d46c13437054da497e254fe9f447d55 is 175, key is test_row_0/A:col10/1733732103043/Put/seqid=0 2024-12-09T08:15:03,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742045_1221 (size=39949) 2024-12-09T08:15:03,464 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/5d46c13437054da497e254fe9f447d55 2024-12-09T08:15:03,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/cffebbf40df54ee5a9c083b0091dc744 is 50, key is test_row_0/B:col10/1733732103043/Put/seqid=0 2024-12-09T08:15:03,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742046_1222 (size=12301) 2024-12-09T08:15:03,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/cffebbf40df54ee5a9c083b0091dc744 2024-12-09T08:15:03,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/debc74dfd8794c4e97a95f2525204ab8 is 50, key is test_row_0/C:col10/1733732103043/Put/seqid=0 2024-12-09T08:15:03,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742047_1223 (size=12301) 2024-12-09T08:15:03,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41772 deadline: 1733732163691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:03,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:03,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:03,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/debc74dfd8794c4e97a95f2525204ab8 2024-12-09T08:15:03,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/5d46c13437054da497e254fe9f447d55 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/5d46c13437054da497e254fe9f447d55 2024-12-09T08:15:03,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/5d46c13437054da497e254fe9f447d55, entries=200, sequenceid=305, filesize=39.0 K 2024-12-09T08:15:03,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/cffebbf40df54ee5a9c083b0091dc744 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/cffebbf40df54ee5a9c083b0091dc744 2024-12-09T08:15:03,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/cffebbf40df54ee5a9c083b0091dc744, entries=150, sequenceid=305, filesize=12.0 K 2024-12-09T08:15:03,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/debc74dfd8794c4e97a95f2525204ab8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/debc74dfd8794c4e97a95f2525204ab8 2024-12-09T08:15:03,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/debc74dfd8794c4e97a95f2525204ab8, entries=150, sequenceid=305, filesize=12.0 K 2024-12-09T08:15:03,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for c9202617e4fb01f766f444f5e419c460 in 884ms, sequenceid=305, compaction requested=true 2024-12-09T08:15:03,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:15:03,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:03,929 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:03,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:03,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:03,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:03,929 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:03,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:03,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:03,930 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:03,930 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:03,930 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/B is initiating minor compaction (all files) 2024-12-09T08:15:03,930 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/A is initiating minor compaction (all files) 2024-12-09T08:15:03,930 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/B in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:03,930 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/A in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:03,930 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/dc7528745d52491a850dba594b4a0369, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/0995f7c8cd1b4e4a9baadbbae665eef4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/cffebbf40df54ee5a9c083b0091dc744] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=36.5 K 2024-12-09T08:15:03,930 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/71aeb6de437840629b57c2ce86a4c660, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e44e7d54ba574e64ba15927ad99fc33f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/5d46c13437054da497e254fe9f447d55] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=100.5 K 2024-12-09T08:15:03,930 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:03,930 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/71aeb6de437840629b57c2ce86a4c660, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e44e7d54ba574e64ba15927ad99fc33f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/5d46c13437054da497e254fe9f447d55] 2024-12-09T08:15:03,930 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting dc7528745d52491a850dba594b4a0369, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733732100283 2024-12-09T08:15:03,931 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71aeb6de437840629b57c2ce86a4c660, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733732100283 2024-12-09T08:15:03,931 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0995f7c8cd1b4e4a9baadbbae665eef4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733732100291 2024-12-09T08:15:03,931 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e44e7d54ba574e64ba15927ad99fc33f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733732100291 2024-12-09T08:15:03,931 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting cffebbf40df54ee5a9c083b0091dc744, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733732102388 2024-12-09T08:15:03,931 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d46c13437054da497e254fe9f447d55, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733732102388 2024-12-09T08:15:03,941 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:03,942 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#B#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:03,942 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/1e6408d38ecb4db19386b564534520f1 is 50, key is test_row_0/B:col10/1733732103043/Put/seqid=0 2024-12-09T08:15:03,943 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120935434097888a4d5fb650d67e11744b5d_c9202617e4fb01f766f444f5e419c460 store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:03,944 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120935434097888a4d5fb650d67e11744b5d_c9202617e4fb01f766f444f5e419c460, store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:03,945 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120935434097888a4d5fb650d67e11744b5d_c9202617e4fb01f766f444f5e419c460 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:03,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742048_1224 (size=12983) 2024-12-09T08:15:03,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742049_1225 (size=4469) 2024-12-09T08:15:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:04,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:15:04,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:15:04,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:04,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:15:04,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:04,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:15:04,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:04,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a1bd0c8a7efe41a5ab610bb3108dbff3_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732103055/Put/seqid=0 2024-12-09T08:15:04,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742050_1226 (size=14994) 2024-12-09T08:15:04,227 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:04,231 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a1bd0c8a7efe41a5ab610bb3108dbff3_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a1bd0c8a7efe41a5ab610bb3108dbff3_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:04,231 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/9c1d9ec499f848b58885bd9c6cee4fd9, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:04,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/9c1d9ec499f848b58885bd9c6cee4fd9 is 175, key is test_row_0/A:col10/1733732103055/Put/seqid=0 2024-12-09T08:15:04,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742051_1227 (size=39949) 2024-12-09T08:15:04,236 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/9c1d9ec499f848b58885bd9c6cee4fd9 2024-12-09T08:15:04,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/4f339af501ed4f79a9f9b1709f9db250 is 50, key is test_row_0/B:col10/1733732103055/Put/seqid=0 2024-12-09T08:15:04,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742052_1228 (size=12301) 2024-12-09T08:15:04,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732104244, max timestamp=1733732104286Num uniques 859; 2024-12-09T08:15:04,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:04,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732104244, max timestamp=1733732104286Num uniques 859; 2024-12-09T08:15:04,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732104244, max timestamp=1733732104286Num uniques 859; 2024-12-09T08:15:04,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:04,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:04,331 INFO [Thread-715 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-715. 2024-12-09T08:15:04,331 DEBUG [Thread-715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:04,331 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,331 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:04,332 INFO [Thread-704 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-704. 2024-12-09T08:15:04,332 DEBUG [Thread-704 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:04,332 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,333 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:04,333 INFO [Thread-719 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-719. 2024-12-09T08:15:04,333 DEBUG [Thread-719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:04,333 DEBUG [Thread-719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,333 DEBUG [Thread-719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:04,335 INFO [Thread-713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-713. 2024-12-09T08:15:04,335 DEBUG [Thread-713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:04,335 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,335 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:04,337 INFO [Thread-717 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-717. 2024-12-09T08:15:04,337 DEBUG [Thread-717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:04,337 DEBUG [Thread-717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,337 DEBUG [Thread-717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:04,359 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#A#compaction#189 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:04,359 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/ce7d6a806ff64c09bd42ee6cd2ee4591 is 175, key is test_row_0/A:col10/1733732103043/Put/seqid=0 2024-12-09T08:15:04,360 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/1e6408d38ecb4db19386b564534520f1 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/1e6408d38ecb4db19386b564534520f1 2024-12-09T08:15:04,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742053_1229 (size=31937) 2024-12-09T08:15:04,365 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9202617e4fb01f766f444f5e419c460/B of c9202617e4fb01f766f444f5e419c460 into 1e6408d38ecb4db19386b564534520f1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:04,365 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:15:04,365 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/B, priority=13, startTime=1733732103929; duration=0sec 2024-12-09T08:15:04,365 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:04,365 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:B 2024-12-09T08:15:04,365 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:04,366 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:04,366 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): c9202617e4fb01f766f444f5e419c460/C is initiating minor compaction (all files) 2024-12-09T08:15:04,366 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9202617e4fb01f766f444f5e419c460/C in TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:04,367 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/677dfc1c1ab141e381e57653c0c5c7f3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/fa63c20267544db68818cd747a1194a3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/debc74dfd8794c4e97a95f2525204ab8] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp, totalSize=36.5 K 2024-12-09T08:15:04,367 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 677dfc1c1ab141e381e57653c0c5c7f3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733732100283 2024-12-09T08:15:04,367 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting fa63c20267544db68818cd747a1194a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733732100291 2024-12-09T08:15:04,367 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting debc74dfd8794c4e97a95f2525204ab8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733732102388 2024-12-09T08:15:04,374 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9202617e4fb01f766f444f5e419c460#C#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:04,374 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/c38354cf73044304bc7be33006a69ef1 is 50, key is test_row_0/C:col10/1733732103043/Put/seqid=0 2024-12-09T08:15:04,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742054_1230 (size=12983) 2024-12-09T08:15:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T08:15:04,431 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:04,462 INFO [Thread-708 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-708. 2024-12-09T08:15:04,463 DEBUG [Thread-708 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:04,463 DEBUG [Thread-708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,463 DEBUG [Thread-708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:04,463 INFO [Thread-710 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-710. 2024-12-09T08:15:04,463 DEBUG [Thread-710 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:04,463 DEBUG [Thread-710 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,463 DEBUG [Thread-710 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:04,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:04,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41778 deadline: 1733732164471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:04,472 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in c9202617e4fb01f766f444f5e419c460 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:04,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41802 deadline: 1733732164491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:04,491 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in c9202617e4fb01f766f444f5e419c460 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:04,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/4f339af501ed4f79a9f9b1709f9db250 2024-12-09T08:15:04,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/dd7bc7c33d2c469f9781a99cd9ab08f7 is 50, key is test_row_0/C:col10/1733732103055/Put/seqid=0 2024-12-09T08:15:04,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742055_1231 (size=12301) 2024-12-09T08:15:04,769 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/ce7d6a806ff64c09bd42ee6cd2ee4591 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/ce7d6a806ff64c09bd42ee6cd2ee4591 2024-12-09T08:15:04,774 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9202617e4fb01f766f444f5e419c460/A of c9202617e4fb01f766f444f5e419c460 into ce7d6a806ff64c09bd42ee6cd2ee4591(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:04,774 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:15:04,774 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/A, priority=13, startTime=1733732103928; duration=0sec 2024-12-09T08:15:04,775 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:04,775 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:A 2024-12-09T08:15:04,783 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/c38354cf73044304bc7be33006a69ef1 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c38354cf73044304bc7be33006a69ef1 2024-12-09T08:15:04,789 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9202617e4fb01f766f444f5e419c460/C of c9202617e4fb01f766f444f5e419c460 into c38354cf73044304bc7be33006a69ef1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:04,789 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:15:04,789 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., storeName=c9202617e4fb01f766f444f5e419c460/C, priority=13, startTime=1733732103929; duration=0sec 2024-12-09T08:15:04,789 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:04,789 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:C 2024-12-09T08:15:05,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/dd7bc7c33d2c469f9781a99cd9ab08f7 2024-12-09T08:15:05,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/9c1d9ec499f848b58885bd9c6cee4fd9 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/9c1d9ec499f848b58885bd9c6cee4fd9 2024-12-09T08:15:05,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/9c1d9ec499f848b58885bd9c6cee4fd9, entries=200, sequenceid=316, filesize=39.0 K 2024-12-09T08:15:05,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/4f339af501ed4f79a9f9b1709f9db250 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/4f339af501ed4f79a9f9b1709f9db250 2024-12-09T08:15:05,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/4f339af501ed4f79a9f9b1709f9db250, entries=150, sequenceid=316, filesize=12.0 K 2024-12-09T08:15:05,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/dd7bc7c33d2c469f9781a99cd9ab08f7 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/dd7bc7c33d2c469f9781a99cd9ab08f7 2024-12-09T08:15:05,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/dd7bc7c33d2c469f9781a99cd9ab08f7, entries=150, sequenceid=316, filesize=12.0 K 2024-12-09T08:15:05,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c9202617e4fb01f766f444f5e419c460 in 873ms, sequenceid=316, compaction requested=false 2024-12-09T08:15:05,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:15:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:08,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T08:15:08,494 INFO [Thread-706 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-706. 2024-12-09T08:15:08,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:15:08,495 DEBUG [Thread-706 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:08,495 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:08,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:08,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:15:08,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:08,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:15:08,495 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:08,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:08,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:08,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120909e6371adab942e0b9829ec95c1a1503_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_0/A:col10/1733732108492/Put/seqid=0 2024-12-09T08:15:08,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742056_1232 (size=12454) 2024-12-09T08:15:08,513 INFO [Thread-702 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-702. 2024-12-09T08:15:08,513 DEBUG [Thread-702 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:08,513 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:08,513 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:08,513 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4567 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4426 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1947 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5834 rows 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1936 2024-12-09T08:15:08,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5798 rows 2024-12-09T08:15:08,514 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T08:15:08,514 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:15:08,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:08,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:08,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:08,515 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T08:15:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T08:15:08,518 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732108518"}]},"ts":"1733732108518"} 2024-12-09T08:15:08,519 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T08:15:08,519 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T08:15:08,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T08:15:08,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, UNASSIGN}] 2024-12-09T08:15:08,523 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, UNASSIGN 2024-12-09T08:15:08,523 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=c9202617e4fb01f766f444f5e419c460, regionState=CLOSING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:15:08,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, UNASSIGN because future has completed 2024-12-09T08:15:08,525 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:15:08,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:15:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T08:15:08,678 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:08,678 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:15:08,678 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing c9202617e4fb01f766f444f5e419c460, disabling compactions & flushes 2024-12-09T08:15:08,678 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T08:15:08,913 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:08,918 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120909e6371adab942e0b9829ec95c1a1503_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120909e6371adab942e0b9829ec95c1a1503_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:08,919 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/2b4f68a274f44b708f0bc7639fe27c41, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:08,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/2b4f68a274f44b708f0bc7639fe27c41 is 175, key is test_row_0/A:col10/1733732108492/Put/seqid=0 2024-12-09T08:15:08,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742057_1233 (size=31255) 2024-12-09T08:15:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T08:15:09,326 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/2b4f68a274f44b708f0bc7639fe27c41 2024-12-09T08:15:09,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/6c3a0643e8c94d10b76afaf4de6748ce is 50, key is test_row_0/B:col10/1733732108492/Put/seqid=0 2024-12-09T08:15:09,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742058_1234 (size=12301) 2024-12-09T08:15:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T08:15:09,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/6c3a0643e8c94d10b76afaf4de6748ce 2024-12-09T08:15:09,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/c9534980cee644649c2636c400b4e25e is 50, key is test_row_0/C:col10/1733732108492/Put/seqid=0 2024-12-09T08:15:09,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742059_1235 (size=12301) 2024-12-09T08:15:10,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/c9534980cee644649c2636c400b4e25e 2024-12-09T08:15:10,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/2b4f68a274f44b708f0bc7639fe27c41 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/2b4f68a274f44b708f0bc7639fe27c41 2024-12-09T08:15:10,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/2b4f68a274f44b708f0bc7639fe27c41, entries=150, sequenceid=355, filesize=30.5 K 2024-12-09T08:15:10,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/6c3a0643e8c94d10b76afaf4de6748ce as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/6c3a0643e8c94d10b76afaf4de6748ce 2024-12-09T08:15:10,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/6c3a0643e8c94d10b76afaf4de6748ce, entries=150, sequenceid=355, filesize=12.0 K 2024-12-09T08:15:10,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/c9534980cee644649c2636c400b4e25e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c9534980cee644649c2636c400b4e25e 2024-12-09T08:15:10,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c9534980cee644649c2636c400b4e25e, entries=150, sequenceid=355, filesize=12.0 K 2024-12-09T08:15:10,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=6.71 KB/6870 for c9202617e4fb01f766f444f5e419c460 in 1683ms, sequenceid=355, compaction requested=true 2024-12-09T08:15:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9202617e4fb01f766f444f5e419c460: 2024-12-09T08:15:10,177 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:10,177 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:10,177 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. because compaction request was cancelled 2024-12-09T08:15:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:10,177 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:A 2024-12-09T08:15:10,177 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. after waiting 0 ms 2024-12-09T08:15:10,177 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:B, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:10,177 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. because compaction request was cancelled 2024-12-09T08:15:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:10,177 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:B 2024-12-09T08:15:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9202617e4fb01f766f444f5e419c460:C, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:10,177 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. because compaction request was cancelled 2024-12-09T08:15:10,177 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing c9202617e4fb01f766f444f5e419c460 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-12-09T08:15:10,177 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9202617e4fb01f766f444f5e419c460:C 2024-12-09T08:15:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:10,178 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=A 2024-12-09T08:15:10,178 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:10,178 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=B 2024-12-09T08:15:10,178 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:10,178 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9202617e4fb01f766f444f5e419c460, store=C 2024-12-09T08:15:10,178 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:10,182 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b86caf6a5f094ce885b53d48319174b3_c9202617e4fb01f766f444f5e419c460 is 50, key is test_row_1/A:col10/1733732108512/Put/seqid=0 2024-12-09T08:15:10,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742060_1236 (size=7374) 2024-12-09T08:15:10,586 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:10,590 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b86caf6a5f094ce885b53d48319174b3_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b86caf6a5f094ce885b53d48319174b3_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:10,591 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/4b23c50fe0884681badc8eeb755a20cc, store: [table=TestAcidGuarantees family=A region=c9202617e4fb01f766f444f5e419c460] 2024-12-09T08:15:10,591 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/4b23c50fe0884681badc8eeb755a20cc is 175, key is test_row_1/A:col10/1733732108512/Put/seqid=0 2024-12-09T08:15:10,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742061_1237 (size=13865) 2024-12-09T08:15:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T08:15:10,997 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/4b23c50fe0884681badc8eeb755a20cc 2024-12-09T08:15:11,012 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/f3d5c209bb2044339ea5a16120e256be is 50, key is test_row_1/B:col10/1733732108512/Put/seqid=0 2024-12-09T08:15:11,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742062_1238 (size=7415) 2024-12-09T08:15:11,418 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/f3d5c209bb2044339ea5a16120e256be 2024-12-09T08:15:11,433 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/ce010bde4bb545748135dbd26a924874 is 50, key is test_row_1/C:col10/1733732108512/Put/seqid=0 2024-12-09T08:15:11,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742063_1239 (size=7415) 2024-12-09T08:15:11,838 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/ce010bde4bb545748135dbd26a924874 2024-12-09T08:15:11,849 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/A/4b23c50fe0884681badc8eeb755a20cc as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/4b23c50fe0884681badc8eeb755a20cc 2024-12-09T08:15:11,854 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/4b23c50fe0884681badc8eeb755a20cc, entries=50, sequenceid=359, filesize=13.5 K 2024-12-09T08:15:11,855 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/B/f3d5c209bb2044339ea5a16120e256be as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/f3d5c209bb2044339ea5a16120e256be 2024-12-09T08:15:11,860 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/f3d5c209bb2044339ea5a16120e256be, entries=50, sequenceid=359, filesize=7.2 K 2024-12-09T08:15:11,861 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/.tmp/C/ce010bde4bb545748135dbd26a924874 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/ce010bde4bb545748135dbd26a924874 2024-12-09T08:15:11,866 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/ce010bde4bb545748135dbd26a924874, entries=50, sequenceid=359, filesize=7.2 K 2024-12-09T08:15:11,867 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for c9202617e4fb01f766f444f5e419c460 in 1689ms, sequenceid=359, compaction requested=true 2024-12-09T08:15:11,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/a40371302fad4c26b54dc1a954a49b87, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f5b9cb2018c54ba791875f03f5ecf677, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d2d5fc177a124a418267376765374b98, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/4082411b22c647499dac3f6d92bfd743, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e603db79e3a04427bfbcf4f787404c9b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/77c64fac696a412c953f3ad83018452a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/508ab1dc41f94324ab09f41e62749e7b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e7ea6d55204746f3ab394b1bdb9edf63, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/8c2996c05a3f43788ca2ba2588090f45, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/27fe3069aeb74133a0762630bd399ba6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d0f9a9744660484b8cd14088cf55dc18, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/32a79536f6484b34b202a04a1701ff5f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/0a0f1ae1dde44aa38c04cf325bce2ccd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e084d878a6c14aaa9cf14ad07eab9113, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/093304ec3c854b7db3cc958c35447a4a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f2555a2f23334c04860c993512de1135, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/c269b84b6e53480c94fecc63cf71333b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/71aeb6de437840629b57c2ce86a4c660, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e44e7d54ba574e64ba15927ad99fc33f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/5d46c13437054da497e254fe9f447d55] to archive 2024-12-09T08:15:11,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:15:11,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/a40371302fad4c26b54dc1a954a49b87 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/a40371302fad4c26b54dc1a954a49b87 2024-12-09T08:15:11,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f5b9cb2018c54ba791875f03f5ecf677 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f5b9cb2018c54ba791875f03f5ecf677 2024-12-09T08:15:11,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d2d5fc177a124a418267376765374b98 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d2d5fc177a124a418267376765374b98 2024-12-09T08:15:11,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/4082411b22c647499dac3f6d92bfd743 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/4082411b22c647499dac3f6d92bfd743 2024-12-09T08:15:11,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e603db79e3a04427bfbcf4f787404c9b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e603db79e3a04427bfbcf4f787404c9b 2024-12-09T08:15:11,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/77c64fac696a412c953f3ad83018452a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/77c64fac696a412c953f3ad83018452a 2024-12-09T08:15:11,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/508ab1dc41f94324ab09f41e62749e7b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/508ab1dc41f94324ab09f41e62749e7b 2024-12-09T08:15:11,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e7ea6d55204746f3ab394b1bdb9edf63 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e7ea6d55204746f3ab394b1bdb9edf63 2024-12-09T08:15:11,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/8c2996c05a3f43788ca2ba2588090f45 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/8c2996c05a3f43788ca2ba2588090f45 2024-12-09T08:15:11,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/27fe3069aeb74133a0762630bd399ba6 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/27fe3069aeb74133a0762630bd399ba6 2024-12-09T08:15:11,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d0f9a9744660484b8cd14088cf55dc18 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/d0f9a9744660484b8cd14088cf55dc18 2024-12-09T08:15:11,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/32a79536f6484b34b202a04a1701ff5f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/32a79536f6484b34b202a04a1701ff5f 2024-12-09T08:15:11,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/0a0f1ae1dde44aa38c04cf325bce2ccd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/0a0f1ae1dde44aa38c04cf325bce2ccd 2024-12-09T08:15:11,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e084d878a6c14aaa9cf14ad07eab9113 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e084d878a6c14aaa9cf14ad07eab9113 2024-12-09T08:15:11,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/093304ec3c854b7db3cc958c35447a4a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/093304ec3c854b7db3cc958c35447a4a 2024-12-09T08:15:11,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f2555a2f23334c04860c993512de1135 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/f2555a2f23334c04860c993512de1135 2024-12-09T08:15:11,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/c269b84b6e53480c94fecc63cf71333b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/c269b84b6e53480c94fecc63cf71333b 2024-12-09T08:15:11,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/71aeb6de437840629b57c2ce86a4c660 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/71aeb6de437840629b57c2ce86a4c660 2024-12-09T08:15:11,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e44e7d54ba574e64ba15927ad99fc33f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/e44e7d54ba574e64ba15927ad99fc33f 2024-12-09T08:15:11,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/5d46c13437054da497e254fe9f447d55 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/5d46c13437054da497e254fe9f447d55 2024-12-09T08:15:11,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/bd7914d17c8e4626adb2a9074616ffdc, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/ca8c0741a39149e5bc6a63b306bbec5c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/2662b2ae502e417380c272f410bc3fde, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/37eaaa72371b4c2b811d713640623e45, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/151e4c99ba044393a1450b52e0acaeab, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/18d61690743b4d218de0c5ae9e49204b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/2c8ae9b4cc7b4c429e55b1815785f521, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/d0a898b232ab4f4691c1ed594423d449, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/82f4d4ace4734a238f4c3cb7f141f9b1, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/d84510e0ccac4cfe87ecc10ac516b2f3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/5e1bc0e89f2e412aa5ae47bb6faef1fb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/25bee722fb954b36adbed4ef4750202c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/6f6afc8706e341e5b0c9c85a7d6919e2, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/c77a9960b5eb4369a040d4cfbbc928a5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/4a16aa6091764e2796c8b124906f1f50, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/62f81c854b0f473a95b052b2230b7814, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/dc7528745d52491a850dba594b4a0369, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/e92d7ac8eed541b39060d585dfb9475b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/0995f7c8cd1b4e4a9baadbbae665eef4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/cffebbf40df54ee5a9c083b0091dc744] to archive 2024-12-09T08:15:11,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:15:11,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/bd7914d17c8e4626adb2a9074616ffdc to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/bd7914d17c8e4626adb2a9074616ffdc 2024-12-09T08:15:11,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/ca8c0741a39149e5bc6a63b306bbec5c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/ca8c0741a39149e5bc6a63b306bbec5c 2024-12-09T08:15:11,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/2662b2ae502e417380c272f410bc3fde to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/2662b2ae502e417380c272f410bc3fde 2024-12-09T08:15:11,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/37eaaa72371b4c2b811d713640623e45 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/37eaaa72371b4c2b811d713640623e45 2024-12-09T08:15:11,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/151e4c99ba044393a1450b52e0acaeab to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/151e4c99ba044393a1450b52e0acaeab 2024-12-09T08:15:11,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/18d61690743b4d218de0c5ae9e49204b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/18d61690743b4d218de0c5ae9e49204b 2024-12-09T08:15:11,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/2c8ae9b4cc7b4c429e55b1815785f521 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/2c8ae9b4cc7b4c429e55b1815785f521 2024-12-09T08:15:11,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/d0a898b232ab4f4691c1ed594423d449 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/d0a898b232ab4f4691c1ed594423d449 2024-12-09T08:15:11,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/82f4d4ace4734a238f4c3cb7f141f9b1 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/82f4d4ace4734a238f4c3cb7f141f9b1 2024-12-09T08:15:11,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/d84510e0ccac4cfe87ecc10ac516b2f3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/d84510e0ccac4cfe87ecc10ac516b2f3 2024-12-09T08:15:11,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/5e1bc0e89f2e412aa5ae47bb6faef1fb to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/5e1bc0e89f2e412aa5ae47bb6faef1fb 2024-12-09T08:15:11,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/25bee722fb954b36adbed4ef4750202c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/25bee722fb954b36adbed4ef4750202c 2024-12-09T08:15:11,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/6f6afc8706e341e5b0c9c85a7d6919e2 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/6f6afc8706e341e5b0c9c85a7d6919e2 2024-12-09T08:15:11,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/c77a9960b5eb4369a040d4cfbbc928a5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/c77a9960b5eb4369a040d4cfbbc928a5 2024-12-09T08:15:11,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/4a16aa6091764e2796c8b124906f1f50 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/4a16aa6091764e2796c8b124906f1f50 2024-12-09T08:15:11,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/62f81c854b0f473a95b052b2230b7814 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/62f81c854b0f473a95b052b2230b7814 2024-12-09T08:15:11,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/dc7528745d52491a850dba594b4a0369 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/dc7528745d52491a850dba594b4a0369 2024-12-09T08:15:11,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/e92d7ac8eed541b39060d585dfb9475b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/e92d7ac8eed541b39060d585dfb9475b 2024-12-09T08:15:11,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/0995f7c8cd1b4e4a9baadbbae665eef4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/0995f7c8cd1b4e4a9baadbbae665eef4 2024-12-09T08:15:11,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/cffebbf40df54ee5a9c083b0091dc744 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/cffebbf40df54ee5a9c083b0091dc744 2024-12-09T08:15:11,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/8e71d0ba91ef448bbe8d0f84c015ebcf, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/146dbd5b557c490293c919b52d0a0837, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/887aa7927af44cba9b20348844a9c036, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c8e9c17f51aa4eda8aba99cf87181607, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/2b48ecde64af4cdb9cd6fb25a82bc9af, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/4b4dcb65f0c94e9092ad07680dc4d42a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/599d8cec0d0346118401cb0dae52814f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/565e341df67b431e9fe2c4feaafba876, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/b924fc7c619844eeb058930423cb25ab, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/5807cc2b7e3441b880f20537cdb3f56b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/ac9de0dc15314befb77569348f8400dd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/e34ebbba083f4620972f8b0031981b00, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/279c63ec956e43649379a4479efd0356, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/9ed5325afd784a49b4591d68ddc81b58, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/05a0dcfacc004a208ab31af99c29a922, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/46c16d78306942e19c43c5b10d1fefd8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/677dfc1c1ab141e381e57653c0c5c7f3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/1cd49ce8b3d0469ba6fca90b27eae989, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/fa63c20267544db68818cd747a1194a3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/debc74dfd8794c4e97a95f2525204ab8] to archive 2024-12-09T08:15:11,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:15:11,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/8e71d0ba91ef448bbe8d0f84c015ebcf to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/8e71d0ba91ef448bbe8d0f84c015ebcf 2024-12-09T08:15:11,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/146dbd5b557c490293c919b52d0a0837 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/146dbd5b557c490293c919b52d0a0837 2024-12-09T08:15:11,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/887aa7927af44cba9b20348844a9c036 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/887aa7927af44cba9b20348844a9c036 2024-12-09T08:15:11,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c8e9c17f51aa4eda8aba99cf87181607 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c8e9c17f51aa4eda8aba99cf87181607 2024-12-09T08:15:11,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/2b48ecde64af4cdb9cd6fb25a82bc9af to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/2b48ecde64af4cdb9cd6fb25a82bc9af 2024-12-09T08:15:11,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/4b4dcb65f0c94e9092ad07680dc4d42a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/4b4dcb65f0c94e9092ad07680dc4d42a 2024-12-09T08:15:11,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/599d8cec0d0346118401cb0dae52814f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/599d8cec0d0346118401cb0dae52814f 2024-12-09T08:15:11,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/565e341df67b431e9fe2c4feaafba876 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/565e341df67b431e9fe2c4feaafba876 2024-12-09T08:15:11,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/b924fc7c619844eeb058930423cb25ab to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/b924fc7c619844eeb058930423cb25ab 2024-12-09T08:15:11,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/5807cc2b7e3441b880f20537cdb3f56b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/5807cc2b7e3441b880f20537cdb3f56b 2024-12-09T08:15:11,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/ac9de0dc15314befb77569348f8400dd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/ac9de0dc15314befb77569348f8400dd 2024-12-09T08:15:11,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/e34ebbba083f4620972f8b0031981b00 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/e34ebbba083f4620972f8b0031981b00 2024-12-09T08:15:11,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/279c63ec956e43649379a4479efd0356 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/279c63ec956e43649379a4479efd0356 2024-12-09T08:15:11,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/9ed5325afd784a49b4591d68ddc81b58 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/9ed5325afd784a49b4591d68ddc81b58 2024-12-09T08:15:11,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/05a0dcfacc004a208ab31af99c29a922 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/05a0dcfacc004a208ab31af99c29a922 2024-12-09T08:15:11,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/46c16d78306942e19c43c5b10d1fefd8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/46c16d78306942e19c43c5b10d1fefd8 2024-12-09T08:15:11,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/677dfc1c1ab141e381e57653c0c5c7f3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/677dfc1c1ab141e381e57653c0c5c7f3 2024-12-09T08:15:11,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/1cd49ce8b3d0469ba6fca90b27eae989 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/1cd49ce8b3d0469ba6fca90b27eae989 2024-12-09T08:15:11,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/fa63c20267544db68818cd747a1194a3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/fa63c20267544db68818cd747a1194a3 2024-12-09T08:15:11,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/debc74dfd8794c4e97a95f2525204ab8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/debc74dfd8794c4e97a95f2525204ab8 2024-12-09T08:15:11,924 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/recovered.edits/362.seqid, newMaxSeqId=362, maxSeqId=4 2024-12-09T08:15:11,924 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. 2024-12-09T08:15:11,924 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for c9202617e4fb01f766f444f5e419c460: Waiting for close lock at 1733732108678Running coprocessor pre-close hooks at 1733732108678Disabling compacts and flushes for region at 1733732108678Disabling writes for close at 1733732110177 (+1499 ms)Obtaining lock to block concurrent updates at 1733732110177Preparing flush snapshotting stores in c9202617e4fb01f766f444f5e419c460 at 1733732110177Finished memstore snapshotting TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1733732110178 (+1 ms)Flushing stores of TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460. at 1733732110178Flushing c9202617e4fb01f766f444f5e419c460/A: creating writer at 1733732110178Flushing mob file c9202617e4fb01f766f444f5e419c460/A: appending metadata at 1733732110182 (+4 ms)Flushing mob file c9202617e4fb01f766f444f5e419c460/A: closing flushed file at 1733732110182Flushing c9202617e4fb01f766f444f5e419c460/A: appending metadata at 1733732110591 (+409 ms)Flushing c9202617e4fb01f766f444f5e419c460/A: closing flushed file at 1733732110591Flushing c9202617e4fb01f766f444f5e419c460/B: creating writer at 1733732111008 (+417 ms)Flushing c9202617e4fb01f766f444f5e419c460/B: appending metadata at 1733732111011 (+3 ms)Flushing c9202617e4fb01f766f444f5e419c460/B: closing flushed file at 1733732111011Flushing c9202617e4fb01f766f444f5e419c460/C: creating writer at 1733732111428 (+417 ms)Flushing c9202617e4fb01f766f444f5e419c460/C: appending metadata at 1733732111432 (+4 ms)Flushing c9202617e4fb01f766f444f5e419c460/C: closing flushed file at 1733732111433 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dbe945b: reopening flushed file at 1733732111848 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31e27e09: reopening flushed file at 1733732111854 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@553e1cfb: reopening flushed file at 1733732111860 (+6 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for c9202617e4fb01f766f444f5e419c460 in 1689ms, sequenceid=359, compaction requested=true at 1733732111867 (+7 ms)Writing region close event to WAL at 1733732111921 (+54 ms)Running coprocessor post-close hooks at 1733732111924 (+3 ms)Closed at 1733732111924 2024-12-09T08:15:11,926 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:11,926 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=c9202617e4fb01f766f444f5e419c460, regionState=CLOSED 2024-12-09T08:15:11,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:15:11,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-09T08:15:11,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure c9202617e4fb01f766f444f5e419c460, server=5bda946059d1,43255,1733732044053 in 3.4030 sec 2024-12-09T08:15:11,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-12-09T08:15:11,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9202617e4fb01f766f444f5e419c460, UNASSIGN in 3.4080 sec 2024-12-09T08:15:11,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-09T08:15:11,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.4110 sec 2024-12-09T08:15:11,934 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732111934"}]},"ts":"1733732111934"} 2024-12-09T08:15:11,935 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T08:15:11,935 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T08:15:11,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.4200 sec 2024-12-09T08:15:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T08:15:12,681 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:12,682 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T08:15:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:12,684 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T08:15:12,684 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:12,687 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,689 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/recovered.edits] 2024-12-09T08:15:12,692 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/2b4f68a274f44b708f0bc7639fe27c41 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/2b4f68a274f44b708f0bc7639fe27c41 2024-12-09T08:15:12,693 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/4b23c50fe0884681badc8eeb755a20cc to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/4b23c50fe0884681badc8eeb755a20cc 2024-12-09T08:15:12,694 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/9c1d9ec499f848b58885bd9c6cee4fd9 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/9c1d9ec499f848b58885bd9c6cee4fd9 2024-12-09T08:15:12,696 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/ce7d6a806ff64c09bd42ee6cd2ee4591 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/A/ce7d6a806ff64c09bd42ee6cd2ee4591 2024-12-09T08:15:12,698 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/1e6408d38ecb4db19386b564534520f1 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/1e6408d38ecb4db19386b564534520f1 2024-12-09T08:15:12,700 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/4f339af501ed4f79a9f9b1709f9db250 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/4f339af501ed4f79a9f9b1709f9db250 2024-12-09T08:15:12,701 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/6c3a0643e8c94d10b76afaf4de6748ce to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/6c3a0643e8c94d10b76afaf4de6748ce 2024-12-09T08:15:12,702 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/f3d5c209bb2044339ea5a16120e256be to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/B/f3d5c209bb2044339ea5a16120e256be 2024-12-09T08:15:12,704 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c38354cf73044304bc7be33006a69ef1 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c38354cf73044304bc7be33006a69ef1 2024-12-09T08:15:12,705 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c9534980cee644649c2636c400b4e25e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/c9534980cee644649c2636c400b4e25e 2024-12-09T08:15:12,707 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/ce010bde4bb545748135dbd26a924874 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/ce010bde4bb545748135dbd26a924874 2024-12-09T08:15:12,708 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/dd7bc7c33d2c469f9781a99cd9ab08f7 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/C/dd7bc7c33d2c469f9781a99cd9ab08f7 2024-12-09T08:15:12,710 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/recovered.edits/362.seqid to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460/recovered.edits/362.seqid 2024-12-09T08:15:12,711 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,711 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T08:15:12,711 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T08:15:12,712 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T08:15:12,716 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120909e6371adab942e0b9829ec95c1a1503_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120909e6371adab942e0b9829ec95c1a1503_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,717 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091bfae47908864dbcb8124f08d72d3191_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091bfae47908864dbcb8124f08d72d3191_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,718 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096f183abc19ed4035a1ffa0b7e1de523a_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096f183abc19ed4035a1ffa0b7e1de523a_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,719 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209888fbf9d97304abeb4f8d44eb80d31fc_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209888fbf9d97304abeb4f8d44eb80d31fc_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,720 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120988e8578e468c45f29ca7197b3a3293f4_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120988e8578e468c45f29ca7197b3a3293f4_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,721 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099b4b65cc0043462c865156be96131851_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099b4b65cc0043462c865156be96131851_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,722 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099c2417b54bc54e7a8b421f82965852bf_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099c2417b54bc54e7a8b421f82965852bf_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,723 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099dc4d2ba481b4769807e3e77405cb71f_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099dc4d2ba481b4769807e3e77405cb71f_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,724 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a1bd0c8a7efe41a5ab610bb3108dbff3_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a1bd0c8a7efe41a5ab610bb3108dbff3_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,725 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab52ddbb87434b52887dbae4a09b82a5_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab52ddbb87434b52887dbae4a09b82a5_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,725 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b86caf6a5f094ce885b53d48319174b3_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b86caf6a5f094ce885b53d48319174b3_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,726 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c5907ad65e6f4d43a61d18fc5e197a63_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c5907ad65e6f4d43a61d18fc5e197a63_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,727 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c7090a33cc524baa82325770ded3d894_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c7090a33cc524baa82325770ded3d894_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,728 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209cf1e6c18dbba425d8241439e7bb373ca_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209cf1e6c18dbba425d8241439e7bb373ca_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,728 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e181e1a33997410f9aa51038cad15cc0_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e181e1a33997410f9aa51038cad15cc0_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,729 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e2e73e0a97884db196c3bc95027b65d3_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e2e73e0a97884db196c3bc95027b65d3_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,730 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f52caa0e21114aa9854c0e3283e6061a_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f52caa0e21114aa9854c0e3283e6061a_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,731 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f790f468c25f4fec98f69c46b1472a6d_c9202617e4fb01f766f444f5e419c460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f790f468c25f4fec98f69c46b1472a6d_c9202617e4fb01f766f444f5e419c460 2024-12-09T08:15:12,731 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T08:15:12,733 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:12,735 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T08:15:12,737 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T08:15:12,739 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:12,739 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T08:15:12,739 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733732112739"}]},"ts":"9223372036854775807"} 2024-12-09T08:15:12,741 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T08:15:12,741 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c9202617e4fb01f766f444f5e419c460, NAME => 'TestAcidGuarantees,,1733732081131.c9202617e4fb01f766f444f5e419c460.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T08:15:12,741 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T08:15:12,741 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733732112741"}]},"ts":"9223372036854775807"} 2024-12-09T08:15:12,743 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T08:15:12,744 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:12,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-12-09T08:15:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T08:15:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T08:15:12,791 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:12,803 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_646891914_22 at /127.0.0.1:58994 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/cluster_81f161a3-853c-b828-244c-d29c3e10103d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/cluster_81f161a3-853c-b828-244c-d29c3e10103d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=395 (was 386) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4801 (was 4024) - AvailableMemoryMB LEAK? - 2024-12-09T08:15:12,813 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=395, ProcessCount=11, AvailableMemoryMB=4797 2024-12-09T08:15:12,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T08:15:12,815 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:15:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:12,817 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T08:15:12,817 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:12,817 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-12-09T08:15:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T08:15:12,819 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T08:15:12,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742064_1240 (size=1061) 2024-12-09T08:15:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T08:15:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T08:15:13,228 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447 2024-12-09T08:15:13,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742065_1241 (size=53) 2024-12-09T08:15:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T08:15:13,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:15:13,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d33b04878a7a85043c90681d784a399c, disabling compactions & flushes 2024-12-09T08:15:13,638 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:13,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:13,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. after waiting 0 ms 2024-12-09T08:15:13,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:13,638 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:13,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d33b04878a7a85043c90681d784a399c: Waiting for close lock at 1733732113638Disabling compacts and flushes for region at 1733732113638Disabling writes for close at 1733732113638Writing region close event to WAL at 1733732113638Closed at 1733732113638 2024-12-09T08:15:13,639 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T08:15:13,639 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733732113639"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733732113639"}]},"ts":"1733732113639"} 2024-12-09T08:15:13,641 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T08:15:13,641 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T08:15:13,641 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732113641"}]},"ts":"1733732113641"} 2024-12-09T08:15:13,643 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T08:15:13,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d33b04878a7a85043c90681d784a399c, ASSIGN}] 2024-12-09T08:15:13,644 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d33b04878a7a85043c90681d784a399c, ASSIGN 2024-12-09T08:15:13,645 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d33b04878a7a85043c90681d784a399c, ASSIGN; state=OFFLINE, location=5bda946059d1,43255,1733732044053; forceNewPlan=false, retain=false 2024-12-09T08:15:13,686 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T08:15:13,795 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=d33b04878a7a85043c90681d784a399c, regionState=OPENING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:15:13,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d33b04878a7a85043c90681d784a399c, ASSIGN because future has completed 2024-12-09T08:15:13,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:15:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T08:15:13,956 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:13,957 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:15:13,957 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,957 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:15:13,958 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,958 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,960 INFO [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,962 INFO [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:15:13,962 INFO [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d33b04878a7a85043c90681d784a399c columnFamilyName A 2024-12-09T08:15:13,963 DEBUG [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:13,963 INFO [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] regionserver.HStore(327): Store=d33b04878a7a85043c90681d784a399c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:15:13,964 INFO [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,965 INFO [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:15:13,966 INFO [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d33b04878a7a85043c90681d784a399c columnFamilyName B 2024-12-09T08:15:13,966 DEBUG [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:13,966 INFO [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] regionserver.HStore(327): Store=d33b04878a7a85043c90681d784a399c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:15:13,967 INFO [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,968 INFO [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:15:13,969 INFO [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d33b04878a7a85043c90681d784a399c columnFamilyName C 2024-12-09T08:15:13,969 DEBUG [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:13,969 INFO [StoreOpener-d33b04878a7a85043c90681d784a399c-1 {}] regionserver.HStore(327): Store=d33b04878a7a85043c90681d784a399c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:15:13,969 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:13,969 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,970 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,971 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,972 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,972 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,973 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:15:13,974 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,977 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:15:13,978 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened d33b04878a7a85043c90681d784a399c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64112819, jitterRate=-0.0446445494890213}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:15:13,978 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:13,979 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for d33b04878a7a85043c90681d784a399c: Running coprocessor pre-open hook at 1733732113958Writing region info on filesystem at 1733732113958Initializing all the Stores at 1733732113959 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732113960 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732113960Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732113960Cleaning up temporary data from old regions at 1733732113972 (+12 ms)Running coprocessor post-open hooks at 1733732113978 (+6 ms)Region opened successfully at 1733732113979 (+1 ms) 2024-12-09T08:15:13,980 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., pid=59, masterSystemTime=1733732113949 2024-12-09T08:15:13,983 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:13,983 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:13,984 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=d33b04878a7a85043c90681d784a399c, regionState=OPEN, openSeqNum=2, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:15:13,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:15:13,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-09T08:15:13,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 in 190 msec 2024-12-09T08:15:13,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-09T08:15:13,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d33b04878a7a85043c90681d784a399c, ASSIGN in 345 msec 2024-12-09T08:15:13,991 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T08:15:13,991 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732113991"}]},"ts":"1733732113991"} 2024-12-09T08:15:13,992 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T08:15:13,993 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T08:15:13,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-12-09T08:15:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T08:15:14,962 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:14,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:14,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:14,964 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b1c70f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:14,965 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@456abfc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,966 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:14,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:14,968 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T08:15:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 5bda946059d1,45611,1733732043366 2024-12-09T08:15:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f7db9b8 2024-12-09T08:15:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T08:15:14,970 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T08:15:14,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:14,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:14,971 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:14,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56cf370f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:14,973 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540df0c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:14,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:14,974 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746f89a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:14,975 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666562db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,976 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:14,976 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:14,977 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0de2a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:14,978 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e89e6bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:14,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:14,979 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@506b0e0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:14,981 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fcf4b83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:14,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:14,982 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c7528d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:14,984 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b45b03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:14,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:14,985 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7387a28e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:14,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b277e71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:14,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:14,989 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@777d45d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:14,990 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ceb430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:14,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:14,991 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@778c3b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:14,992 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3394db16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:14,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:14,994 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64bbdaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:14,995 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a3f408, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:14,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:14,996 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ed4278, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:14,998 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dbe816f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:14,999 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:14,999 DEBUG [Thread-1141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:14,999 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:15,000 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:15,000 DEBUG [Thread-1135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:15,000 DEBUG [Thread-1139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:15,000 DEBUG [Thread-1133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:15,000 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:15,000 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-09T08:15:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:15,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:15,002 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:15,002 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T08:15:15,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:15,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:15,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:15,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:15,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:15:15,004 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:15,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:15,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:15:15,005 DEBUG [Thread-1137 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:15,005 DEBUG [Thread-1137 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:15:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:15,007 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:15,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:15:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:15:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:15:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:15:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:15:15,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:15,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:15:15,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:15:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:15,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:15:15,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:15,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:15,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:15,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:15,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:15,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:15,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/9bb5966dd3b347a1a0aa127a036331c5 is 50, key is test_row_0/A:col10/1733732115020/Put/seqid=0 2024-12-09T08:15:15,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732175047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732175049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732175050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732175055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732175056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742066_1242 (size=14341) 2024-12-09T08:15:15,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/9bb5966dd3b347a1a0aa127a036331c5 2024-12-09T08:15:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T08:15:15,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/140b40dc6ae641cd80c7408f92c023c2 is 50, key is test_row_0/B:col10/1733732115020/Put/seqid=0 2024-12-09T08:15:15,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T08:15:15,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:15,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:15,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:15,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732175161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732175162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732175171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732175171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732175171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742067_1243 (size=12001) 2024-12-09T08:15:15,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T08:15:15,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:15,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:15,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:15,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T08:15:15,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732175371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732175381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732175381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732175381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732175382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T08:15:15,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:15,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:15,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:15,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/140b40dc6ae641cd80c7408f92c023c2 2024-12-09T08:15:15,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T08:15:15,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:15,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:15,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:15,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/0f183aaac9f942218266d1c2606c95df is 50, key is test_row_0/C:col10/1733732115020/Put/seqid=0 2024-12-09T08:15:15,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742068_1244 (size=12001) 2024-12-09T08:15:15,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/0f183aaac9f942218266d1c2606c95df 2024-12-09T08:15:15,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/9bb5966dd3b347a1a0aa127a036331c5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/9bb5966dd3b347a1a0aa127a036331c5 2024-12-09T08:15:15,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/9bb5966dd3b347a1a0aa127a036331c5, entries=200, sequenceid=13, filesize=14.0 K 2024-12-09T08:15:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T08:15:15,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/140b40dc6ae641cd80c7408f92c023c2 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/140b40dc6ae641cd80c7408f92c023c2 2024-12-09T08:15:15,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/140b40dc6ae641cd80c7408f92c023c2, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T08:15:15,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/0f183aaac9f942218266d1c2606c95df as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/0f183aaac9f942218266d1c2606c95df 2024-12-09T08:15:15,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/0f183aaac9f942218266d1c2606c95df, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T08:15:15,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d33b04878a7a85043c90681d784a399c in 644ms, sequenceid=13, compaction requested=false 2024-12-09T08:15:15,667 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T08:15:15,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:15,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T08:15:15,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:15,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:15,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:15,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:15,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:15,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:15,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732175696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732175696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732175696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732175699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732175700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/bd4760e9a1b448aba4e871359d7ce4b3 is 50, key is test_row_0/A:col10/1733732115048/Put/seqid=0 2024-12-09T08:15:15,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742069_1245 (size=14341) 2024-12-09T08:15:15,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/bd4760e9a1b448aba4e871359d7ce4b3 2024-12-09T08:15:15,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/61b5fa7186cc414aa73615dfe7b5d77b is 50, key is test_row_0/B:col10/1733732115048/Put/seqid=0 2024-12-09T08:15:15,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742070_1246 (size=12001) 2024-12-09T08:15:15,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T08:15:15,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:15,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:15,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:15,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732175811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732175811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732175811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732175812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732175812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:15,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T08:15:15,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:15,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:15,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:15,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:16,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732176021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732176021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732176021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732176022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732176022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T08:15:16,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:16,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:16,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:16,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:16,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T08:15:16,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/61b5fa7186cc414aa73615dfe7b5d77b 2024-12-09T08:15:16,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/9c1e645359a74031b96c7a65e01cf5e5 is 50, key is test_row_0/C:col10/1733732115048/Put/seqid=0 2024-12-09T08:15:16,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742071_1247 (size=12001) 2024-12-09T08:15:16,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/9c1e645359a74031b96c7a65e01cf5e5 2024-12-09T08:15:16,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/bd4760e9a1b448aba4e871359d7ce4b3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/bd4760e9a1b448aba4e871359d7ce4b3 2024-12-09T08:15:16,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/bd4760e9a1b448aba4e871359d7ce4b3, entries=200, sequenceid=34, filesize=14.0 K 2024-12-09T08:15:16,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/61b5fa7186cc414aa73615dfe7b5d77b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/61b5fa7186cc414aa73615dfe7b5d77b 2024-12-09T08:15:16,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/61b5fa7186cc414aa73615dfe7b5d77b, entries=150, sequenceid=34, filesize=11.7 K 2024-12-09T08:15:16,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/9c1e645359a74031b96c7a65e01cf5e5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/9c1e645359a74031b96c7a65e01cf5e5 2024-12-09T08:15:16,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/9c1e645359a74031b96c7a65e01cf5e5, entries=150, sequenceid=34, filesize=11.7 K 2024-12-09T08:15:16,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for d33b04878a7a85043c90681d784a399c in 536ms, sequenceid=34, compaction requested=false 2024-12-09T08:15:16,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:16,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T08:15:16,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:16,235 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:15:16,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:16,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:16,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:16,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:16,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:16,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:16,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/1cede18eeb0e4218bbce903b8e26689b is 50, key is test_row_0/A:col10/1733732115699/Put/seqid=0 2024-12-09T08:15:16,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742072_1248 (size=12001) 2024-12-09T08:15:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:16,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:16,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732176346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732176347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732176348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732116344, max timestamp=1733732116349Num uniques 859; 2024-12-09T08:15:16,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732116344, max timestamp=1733732116349Num uniques 859; 2024-12-09T08:15:16,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:16,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:16,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732116344, max timestamp=1733732116349Num uniques 859; 2024-12-09T08:15:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732176384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732176388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732176461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732176461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732176461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732176500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732176501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,659 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/1cede18eeb0e4218bbce903b8e26689b 2024-12-09T08:15:16,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/e3cfd7dfb3464bc49e99417bcab24f68 is 50, key is test_row_0/B:col10/1733732115699/Put/seqid=0 2024-12-09T08:15:16,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742073_1249 (size=12001) 2024-12-09T08:15:16,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732176671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732176671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732176671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732176711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732176711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732176981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732176981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732176981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732177020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732177021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,070 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/e3cfd7dfb3464bc49e99417bcab24f68 2024-12-09T08:15:17,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/4e44b24fb35e4ed98c36450446e6098e is 50, key is test_row_0/C:col10/1733732115699/Put/seqid=0 2024-12-09T08:15:17,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742074_1250 (size=12001) 2024-12-09T08:15:17,097 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/4e44b24fb35e4ed98c36450446e6098e 2024-12-09T08:15:17,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/1cede18eeb0e4218bbce903b8e26689b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/1cede18eeb0e4218bbce903b8e26689b 2024-12-09T08:15:17,107 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/1cede18eeb0e4218bbce903b8e26689b, entries=150, sequenceid=45, filesize=11.7 K 2024-12-09T08:15:17,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/e3cfd7dfb3464bc49e99417bcab24f68 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e3cfd7dfb3464bc49e99417bcab24f68 2024-12-09T08:15:17,112 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e3cfd7dfb3464bc49e99417bcab24f68, entries=150, sequenceid=45, filesize=11.7 K 2024-12-09T08:15:17,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/4e44b24fb35e4ed98c36450446e6098e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/4e44b24fb35e4ed98c36450446e6098e 2024-12-09T08:15:17,119 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/4e44b24fb35e4ed98c36450446e6098e, entries=150, sequenceid=45, filesize=11.7 K 2024-12-09T08:15:17,119 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d33b04878a7a85043c90681d784a399c in 884ms, sequenceid=45, compaction requested=true 2024-12-09T08:15:17,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:17,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:17,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-09T08:15:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-09T08:15:17,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-09T08:15:17,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1170 sec 2024-12-09T08:15:17,137 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.1240 sec 2024-12-09T08:15:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T08:15:17,181 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:17,182 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-09T08:15:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T08:15:17,183 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:17,183 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:17,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T08:15:17,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T08:15:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:17,336 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-09T08:15:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:17,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/e8f9e797c0bc405db1ca0f0fbee59656 is 50, key is test_row_0/A:col10/1733732116383/Put/seqid=0 2024-12-09T08:15:17,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742075_1251 (size=12001) 2024-12-09T08:15:17,443 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:15:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:17,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T08:15:17,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732177531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732177531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732177531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732177531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732177531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732177641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732177641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732177641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,745 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/e8f9e797c0bc405db1ca0f0fbee59656 2024-12-09T08:15:17,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/0e41c4a2a8ac445f8ece62b67160388f is 50, key is test_row_0/B:col10/1733732116383/Put/seqid=0 2024-12-09T08:15:17,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742076_1252 (size=12001) 2024-12-09T08:15:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T08:15:17,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732177851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732177851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732177851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:18,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732178161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:18,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732178161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732178161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:18,164 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/0e41c4a2a8ac445f8ece62b67160388f 2024-12-09T08:15:18,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/d590819bb2b343afb38c480c88e40d8d is 50, key is test_row_0/C:col10/1733732116383/Put/seqid=0 2024-12-09T08:15:18,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742077_1253 (size=12001) 2024-12-09T08:15:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T08:15:18,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732178551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:18,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732178551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:18,553 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in d33b04878a7a85043c90681d784a399c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:18,553 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in d33b04878a7a85043c90681d784a399c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:18,600 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/d590819bb2b343afb38c480c88e40d8d 2024-12-09T08:15:18,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/e8f9e797c0bc405db1ca0f0fbee59656 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/e8f9e797c0bc405db1ca0f0fbee59656 2024-12-09T08:15:18,608 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/e8f9e797c0bc405db1ca0f0fbee59656, entries=150, sequenceid=80, filesize=11.7 K 2024-12-09T08:15:18,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/0e41c4a2a8ac445f8ece62b67160388f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/0e41c4a2a8ac445f8ece62b67160388f 2024-12-09T08:15:18,613 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/0e41c4a2a8ac445f8ece62b67160388f, entries=150, sequenceid=80, filesize=11.7 K 2024-12-09T08:15:18,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/d590819bb2b343afb38c480c88e40d8d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/d590819bb2b343afb38c480c88e40d8d 2024-12-09T08:15:18,617 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/d590819bb2b343afb38c480c88e40d8d, entries=150, sequenceid=80, filesize=11.7 K 2024-12-09T08:15:18,618 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for d33b04878a7a85043c90681d784a399c in 1282ms, sequenceid=80, compaction requested=true 2024-12-09T08:15:18,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:18,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:18,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-09T08:15:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-09T08:15:18,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-09T08:15:18,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-12-09T08:15:18,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.4390 sec 2024-12-09T08:15:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:18,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T08:15:18,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:18,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:18,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:18,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:18,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:18,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:18,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/51f46e8c917d45469140113dc6261352 is 50, key is test_row_0/A:col10/1733732117510/Put/seqid=0 2024-12-09T08:15:18,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742078_1254 (size=12001) 2024-12-09T08:15:18,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732178693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:18,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732178694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:18,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732178698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:18,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732178801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:18,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732178801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:18,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:18,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732178811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:19,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732179011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:19,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732179011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:19,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732179021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:19,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/51f46e8c917d45469140113dc6261352 2024-12-09T08:15:19,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a6336a6337ba48b2804a2d34ed75fefe is 50, key is test_row_0/B:col10/1733732117510/Put/seqid=0 2024-12-09T08:15:19,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742079_1255 (size=12001) 2024-12-09T08:15:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732179320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732179320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:19,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732179331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T08:15:19,341 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:19,342 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-09T08:15:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T08:15:19,343 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:19,344 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:19,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T08:15:19,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T08:15:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:19,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a6336a6337ba48b2804a2d34ed75fefe 2024-12-09T08:15:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:19,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:19,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/766ca32c95284009b7802293ccbc0bbd is 50, key is test_row_0/C:col10/1733732117510/Put/seqid=0 2024-12-09T08:15:19,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742080_1256 (size=12001) 2024-12-09T08:15:19,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T08:15:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:19,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T08:15:19,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T08:15:19,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:19,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:19,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732179831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:19,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732179831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:19,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732179841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:19,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/766ca32c95284009b7802293ccbc0bbd 2024-12-09T08:15:19,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/51f46e8c917d45469140113dc6261352 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/51f46e8c917d45469140113dc6261352 2024-12-09T08:15:19,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/51f46e8c917d45469140113dc6261352, entries=150, sequenceid=93, filesize=11.7 K 2024-12-09T08:15:19,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a6336a6337ba48b2804a2d34ed75fefe as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a6336a6337ba48b2804a2d34ed75fefe 2024-12-09T08:15:19,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a6336a6337ba48b2804a2d34ed75fefe, entries=150, sequenceid=93, filesize=11.7 K 2024-12-09T08:15:19,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/766ca32c95284009b7802293ccbc0bbd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/766ca32c95284009b7802293ccbc0bbd 2024-12-09T08:15:19,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/766ca32c95284009b7802293ccbc0bbd, entries=150, sequenceid=93, filesize=11.7 K 2024-12-09T08:15:19,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d33b04878a7a85043c90681d784a399c in 1259ms, sequenceid=93, compaction requested=true 2024-12-09T08:15:19,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:19,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:19,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:19,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:19,931 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T08:15:19,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:19,931 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T08:15:19,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:19,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:19,932 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64685 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T08:15:19,932 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T08:15:19,933 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/B is initiating minor compaction (all files) 2024-12-09T08:15:19,933 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/A is initiating minor compaction (all files) 2024-12-09T08:15:19,933 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/A in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:19,933 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/B in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:19,933 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/9bb5966dd3b347a1a0aa127a036331c5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/bd4760e9a1b448aba4e871359d7ce4b3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/1cede18eeb0e4218bbce903b8e26689b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/e8f9e797c0bc405db1ca0f0fbee59656, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/51f46e8c917d45469140113dc6261352] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=63.2 K 2024-12-09T08:15:19,933 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/140b40dc6ae641cd80c7408f92c023c2, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/61b5fa7186cc414aa73615dfe7b5d77b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e3cfd7dfb3464bc49e99417bcab24f68, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/0e41c4a2a8ac445f8ece62b67160388f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a6336a6337ba48b2804a2d34ed75fefe] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=58.6 K 2024-12-09T08:15:19,933 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bb5966dd3b347a1a0aa127a036331c5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733732115012 2024-12-09T08:15:19,933 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 140b40dc6ae641cd80c7408f92c023c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733732115017 2024-12-09T08:15:19,933 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd4760e9a1b448aba4e871359d7ce4b3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733732115048 2024-12-09T08:15:19,933 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 61b5fa7186cc414aa73615dfe7b5d77b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733732115048 2024-12-09T08:15:19,933 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cede18eeb0e4218bbce903b8e26689b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733732115695 2024-12-09T08:15:19,933 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting e3cfd7dfb3464bc49e99417bcab24f68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733732115695 2024-12-09T08:15:19,934 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8f9e797c0bc405db1ca0f0fbee59656, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733732116376 2024-12-09T08:15:19,934 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e41c4a2a8ac445f8ece62b67160388f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733732116376 2024-12-09T08:15:19,934 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51f46e8c917d45469140113dc6261352, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733732117510 2024-12-09T08:15:19,934 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a6336a6337ba48b2804a2d34ed75fefe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733732117510 2024-12-09T08:15:19,941 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#A#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:19,942 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/6c3275b6f7a84a6fa9422f90bf2685f4 is 50, key is test_row_0/A:col10/1733732117510/Put/seqid=0 2024-12-09T08:15:19,943 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#B#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:19,943 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/f11fcc1cd4074b30afe2b3e7920efa51 is 50, key is test_row_0/B:col10/1733732117510/Put/seqid=0 2024-12-09T08:15:19,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742081_1257 (size=12173) 2024-12-09T08:15:19,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742082_1258 (size=12173) 2024-12-09T08:15:19,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T08:15:19,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:19,956 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T08:15:19,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:19,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:19,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:19,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:19,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:19,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:19,957 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/f11fcc1cd4074b30afe2b3e7920efa51 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/f11fcc1cd4074b30afe2b3e7920efa51 2024-12-09T08:15:19,957 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/6c3275b6f7a84a6fa9422f90bf2685f4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/6c3275b6f7a84a6fa9422f90bf2685f4 2024-12-09T08:15:19,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/f481c9893ac74c9097a795d033611993 is 50, key is test_row_0/A:col10/1733732118692/Put/seqid=0 2024-12-09T08:15:19,963 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d33b04878a7a85043c90681d784a399c/A of d33b04878a7a85043c90681d784a399c into 6c3275b6f7a84a6fa9422f90bf2685f4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:19,963 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:19,963 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/A, priority=11, startTime=1733732119931; duration=0sec 2024-12-09T08:15:19,964 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:19,964 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:A 2024-12-09T08:15:19,964 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T08:15:19,965 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d33b04878a7a85043c90681d784a399c/B of d33b04878a7a85043c90681d784a399c into f11fcc1cd4074b30afe2b3e7920efa51(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:19,965 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:19,965 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/B, priority=11, startTime=1733732119931; duration=0sec 2024-12-09T08:15:19,965 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:19,965 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:B 2024-12-09T08:15:19,966 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T08:15:19,966 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/C is initiating minor compaction (all files) 2024-12-09T08:15:19,966 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/C in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:19,966 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/0f183aaac9f942218266d1c2606c95df, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/9c1e645359a74031b96c7a65e01cf5e5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/4e44b24fb35e4ed98c36450446e6098e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/d590819bb2b343afb38c480c88e40d8d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/766ca32c95284009b7802293ccbc0bbd] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=58.6 K 2024-12-09T08:15:19,967 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f183aaac9f942218266d1c2606c95df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733732115017 2024-12-09T08:15:19,967 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c1e645359a74031b96c7a65e01cf5e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733732115048 2024-12-09T08:15:19,967 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e44b24fb35e4ed98c36450446e6098e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733732115695 2024-12-09T08:15:19,968 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d590819bb2b343afb38c480c88e40d8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733732116376 2024-12-09T08:15:19,968 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 766ca32c95284009b7802293ccbc0bbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733732117510 2024-12-09T08:15:19,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742083_1259 (size=12001) 2024-12-09T08:15:19,970 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/f481c9893ac74c9097a795d033611993 2024-12-09T08:15:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T08:15:19,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/2a11dceb43d54681a51de185fee9953b is 50, key is test_row_0/B:col10/1733732118692/Put/seqid=0 2024-12-09T08:15:19,986 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#C#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:19,987 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/165bb059dd994912859783a1f41caf74 is 50, key is test_row_0/C:col10/1733732117510/Put/seqid=0 2024-12-09T08:15:19,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742084_1260 (size=12001) 2024-12-09T08:15:19,990 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/2a11dceb43d54681a51de185fee9953b 2024-12-09T08:15:19,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742085_1261 (size=12173) 2024-12-09T08:15:19,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/5af98effa16a43ce87548de796337ebe is 50, key is test_row_0/C:col10/1733732118692/Put/seqid=0 2024-12-09T08:15:20,001 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/165bb059dd994912859783a1f41caf74 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/165bb059dd994912859783a1f41caf74 2024-12-09T08:15:20,006 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d33b04878a7a85043c90681d784a399c/C of d33b04878a7a85043c90681d784a399c into 165bb059dd994912859783a1f41caf74(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:20,006 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:20,006 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/C, priority=11, startTime=1733732119931; duration=0sec 2024-12-09T08:15:20,006 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:20,006 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:C 2024-12-09T08:15:20,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742086_1262 (size=12001) 2024-12-09T08:15:20,010 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/5af98effa16a43ce87548de796337ebe 2024-12-09T08:15:20,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/f481c9893ac74c9097a795d033611993 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/f481c9893ac74c9097a795d033611993 2024-12-09T08:15:20,018 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/f481c9893ac74c9097a795d033611993, entries=150, sequenceid=112, filesize=11.7 K 2024-12-09T08:15:20,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-09T08:15:20,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/2a11dceb43d54681a51de185fee9953b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/2a11dceb43d54681a51de185fee9953b 2024-12-09T08:15:20,023 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/2a11dceb43d54681a51de185fee9953b, entries=150, sequenceid=112, filesize=11.7 K 2024-12-09T08:15:20,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/5af98effa16a43ce87548de796337ebe as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5af98effa16a43ce87548de796337ebe 2024-12-09T08:15:20,028 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5af98effa16a43ce87548de796337ebe, entries=150, sequenceid=112, filesize=11.7 K 2024-12-09T08:15:20,029 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for d33b04878a7a85043c90681d784a399c in 73ms, sequenceid=112, compaction requested=false 2024-12-09T08:15:20,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:20,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:20,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-09T08:15:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-09T08:15:20,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-09T08:15:20,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 686 msec 2024-12-09T08:15:20,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 690 msec 2024-12-09T08:15:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T08:15:20,481 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:20,481 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-09T08:15:20,483 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:20,484 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:20,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T08:15:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:20,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:15:20,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:20,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:20,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:20,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:20,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:20,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:20,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/5ee01ae01b8049d2be05eb8335adfb87 is 50, key is test_row_0/A:col10/1733732120571/Put/seqid=0 2024-12-09T08:15:20,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742087_1263 (size=9657) 2024-12-09T08:15:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T08:15:20,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733732120597, max timestamp=1733732120609Num uniques 859; 2024-12-09T08:15:20,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733732120597, max timestamp=1733732120609Num uniques 859; 2024-12-09T08:15:20,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:20,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:20,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733732120597, max timestamp=1733732120609Num uniques 859; 2024-12-09T08:15:20,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:20,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T08:15:20,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:20,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:20,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:20,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:20,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:20,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732180646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:20,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732180647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:20,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732180751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:20,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732180761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:20,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T08:15:20,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:20,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:20,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:20,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:20,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T08:15:20,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732180841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:20,844 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in d33b04878a7a85043c90681d784a399c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:20,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:20,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732180851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:20,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732180851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:20,855 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in d33b04878a7a85043c90681d784a399c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:20,855 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in d33b04878a7a85043c90681d784a399c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:20,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T08:15:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:20,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:20,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732180961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:20,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732180971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:20,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/5ee01ae01b8049d2be05eb8335adfb87 2024-12-09T08:15:20,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/d178cce8fc2d41d49035689e5122ffe8 is 50, key is test_row_0/B:col10/1733732120571/Put/seqid=0 2024-12-09T08:15:21,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742088_1264 (size=9657) 2024-12-09T08:15:21,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/d178cce8fc2d41d49035689e5122ffe8 2024-12-09T08:15:21,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/2ba257ea779e4a2c8fcf84c980e1a782 is 50, key is test_row_0/C:col10/1733732120571/Put/seqid=0 2024-12-09T08:15:21,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742089_1265 (size=9657) 2024-12-09T08:15:21,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T08:15:21,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:21,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:21,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:21,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:21,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T08:15:21,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T08:15:21,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:21,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:21,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:21,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:21,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:21,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732181271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:21,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732181280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:21,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T08:15:21,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:21,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:21,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:21,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:21,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:21,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/2ba257ea779e4a2c8fcf84c980e1a782 2024-12-09T08:15:21,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/5ee01ae01b8049d2be05eb8335adfb87 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/5ee01ae01b8049d2be05eb8335adfb87 2024-12-09T08:15:21,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/5ee01ae01b8049d2be05eb8335adfb87, entries=100, sequenceid=125, filesize=9.4 K 2024-12-09T08:15:21,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/d178cce8fc2d41d49035689e5122ffe8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/d178cce8fc2d41d49035689e5122ffe8 2024-12-09T08:15:21,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/d178cce8fc2d41d49035689e5122ffe8, entries=100, sequenceid=125, filesize=9.4 K 2024-12-09T08:15:21,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/2ba257ea779e4a2c8fcf84c980e1a782 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/2ba257ea779e4a2c8fcf84c980e1a782 2024-12-09T08:15:21,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/2ba257ea779e4a2c8fcf84c980e1a782, entries=100, sequenceid=125, filesize=9.4 K 2024-12-09T08:15:21,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d33b04878a7a85043c90681d784a399c in 903ms, sequenceid=125, compaction requested=true 2024-12-09T08:15:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:21,476 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:21,476 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:21,477 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:21,477 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:21,477 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/B is initiating minor compaction (all files) 2024-12-09T08:15:21,477 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/A is initiating minor compaction (all files) 2024-12-09T08:15:21,477 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/A in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:21,477 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/6c3275b6f7a84a6fa9422f90bf2685f4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/f481c9893ac74c9097a795d033611993, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/5ee01ae01b8049d2be05eb8335adfb87] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=33.0 K 2024-12-09T08:15:21,477 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/B in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:21,477 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/f11fcc1cd4074b30afe2b3e7920efa51, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/2a11dceb43d54681a51de185fee9953b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/d178cce8fc2d41d49035689e5122ffe8] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=33.0 K 2024-12-09T08:15:21,477 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c3275b6f7a84a6fa9422f90bf2685f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733732117510 2024-12-09T08:15:21,478 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting f11fcc1cd4074b30afe2b3e7920efa51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733732117510 2024-12-09T08:15:21,480 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting f481c9893ac74c9097a795d033611993, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733732118692 2024-12-09T08:15:21,480 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a11dceb43d54681a51de185fee9953b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733732118692 2024-12-09T08:15:21,480 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ee01ae01b8049d2be05eb8335adfb87, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733732120570 2024-12-09T08:15:21,480 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting d178cce8fc2d41d49035689e5122ffe8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733732120570 2024-12-09T08:15:21,487 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#A#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:21,487 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/da488ff561d34591b61d4372a2712bd4 is 50, key is test_row_0/A:col10/1733732120571/Put/seqid=0 2024-12-09T08:15:21,488 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#B#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:21,488 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/0e78547564544574ac4654d11b639070 is 50, key is test_row_0/B:col10/1733732120571/Put/seqid=0 2024-12-09T08:15:21,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742090_1266 (size=12275) 2024-12-09T08:15:21,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742091_1267 (size=12275) 2024-12-09T08:15:21,495 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/da488ff561d34591b61d4372a2712bd4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/da488ff561d34591b61d4372a2712bd4 2024-12-09T08:15:21,500 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/0e78547564544574ac4654d11b639070 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/0e78547564544574ac4654d11b639070 2024-12-09T08:15:21,500 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/A of d33b04878a7a85043c90681d784a399c into da488ff561d34591b61d4372a2712bd4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:21,500 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:21,500 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/A, priority=13, startTime=1733732121476; duration=0sec 2024-12-09T08:15:21,500 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:21,500 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:A 2024-12-09T08:15:21,500 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:21,502 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:21,502 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/C is initiating minor compaction (all files) 2024-12-09T08:15:21,502 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/C in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:21,502 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/165bb059dd994912859783a1f41caf74, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5af98effa16a43ce87548de796337ebe, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/2ba257ea779e4a2c8fcf84c980e1a782] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=33.0 K 2024-12-09T08:15:21,503 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 165bb059dd994912859783a1f41caf74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733732117510 2024-12-09T08:15:21,503 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5af98effa16a43ce87548de796337ebe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733732118692 2024-12-09T08:15:21,503 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ba257ea779e4a2c8fcf84c980e1a782, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733732120570 2024-12-09T08:15:21,506 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/B of d33b04878a7a85043c90681d784a399c into 0e78547564544574ac4654d11b639070(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:21,506 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:21,506 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/B, priority=13, startTime=1733732121476; duration=0sec 2024-12-09T08:15:21,506 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:21,506 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:B 2024-12-09T08:15:21,526 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:21,527 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/e9055227fca54129a1d3ecd96c73b2e9 is 50, key is test_row_0/C:col10/1733732120571/Put/seqid=0 2024-12-09T08:15:21,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742092_1268 (size=12275) 2024-12-09T08:15:21,541 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/e9055227fca54129a1d3ecd96c73b2e9 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/e9055227fca54129a1d3ecd96c73b2e9 2024-12-09T08:15:21,546 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/C of d33b04878a7a85043c90681d784a399c into e9055227fca54129a1d3ecd96c73b2e9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:21,546 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:21,546 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/C, priority=13, startTime=1733732121476; duration=0sec 2024-12-09T08:15:21,546 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:21,546 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:C 2024-12-09T08:15:21,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T08:15:21,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:21,556 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T08:15:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:21,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c99ab3fe9f404e31a3c0c706fa1c5455 is 50, key is test_row_0/A:col10/1733732120641/Put/seqid=0 2024-12-09T08:15:21,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742093_1269 (size=12151) 2024-12-09T08:15:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T08:15:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:21,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:21,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732181800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:21,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732181802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:21,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732181911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:21,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732181911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:21,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:21,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:21,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:21,965 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c99ab3fe9f404e31a3c0c706fa1c5455 2024-12-09T08:15:21,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/1c55e7f9e2d845f5a944835e36f08435 is 50, key is test_row_0/B:col10/1733732120641/Put/seqid=0 2024-12-09T08:15:21,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742094_1270 (size=12151) 2024-12-09T08:15:22,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732182121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:22,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732182121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:22,376 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/1c55e7f9e2d845f5a944835e36f08435 2024-12-09T08:15:22,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/f5edbffd8ebd48e08eb34b46039400b4 is 50, key is test_row_0/C:col10/1733732120641/Put/seqid=0 2024-12-09T08:15:22,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742095_1271 (size=12151) 2024-12-09T08:15:22,394 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/f5edbffd8ebd48e08eb34b46039400b4 2024-12-09T08:15:22,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c99ab3fe9f404e31a3c0c706fa1c5455 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c99ab3fe9f404e31a3c0c706fa1c5455 2024-12-09T08:15:22,404 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c99ab3fe9f404e31a3c0c706fa1c5455, entries=150, sequenceid=164, filesize=11.9 K 2024-12-09T08:15:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/1c55e7f9e2d845f5a944835e36f08435 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/1c55e7f9e2d845f5a944835e36f08435 2024-12-09T08:15:22,409 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/1c55e7f9e2d845f5a944835e36f08435, entries=150, sequenceid=164, filesize=11.9 K 2024-12-09T08:15:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/f5edbffd8ebd48e08eb34b46039400b4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/f5edbffd8ebd48e08eb34b46039400b4 2024-12-09T08:15:22,413 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/f5edbffd8ebd48e08eb34b46039400b4, entries=150, sequenceid=164, filesize=11.9 K 2024-12-09T08:15:22,414 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for d33b04878a7a85043c90681d784a399c in 858ms, sequenceid=164, compaction requested=false 2024-12-09T08:15:22,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:22,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:22,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-09T08:15:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-09T08:15:22,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-09T08:15:22,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9310 sec 2024-12-09T08:15:22,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.9360 sec 2024-12-09T08:15:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:22,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:15:22,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:22,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:22,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:22,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:22,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:22,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:22,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/089a42e3aa1c44b8bec76eccd30c4148 is 50, key is test_row_0/A:col10/1733732121801/Put/seqid=0 2024-12-09T08:15:22,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742096_1272 (size=14541) 2024-12-09T08:15:22,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732122449, max timestamp=1733732122484Num uniques 859; 2024-12-09T08:15:22,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:22,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732122449, max timestamp=1733732122484Num uniques 859; 2024-12-09T08:15:22,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:22,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733732122449, max timestamp=1733732122484Num uniques 859; 2024-12-09T08:15:22,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:22,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:22,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732182523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732182522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T08:15:22,631 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:22,631 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-09T08:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T08:15:22,633 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:22,633 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:22,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:22,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732182631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732182631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T08:15:22,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T08:15:22,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:22,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:22,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:22,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:22,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/089a42e3aa1c44b8bec76eccd30c4148 2024-12-09T08:15:22,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732182841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:22,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732182841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:22,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/9a1f4d334e084fe4b4f70d8e3461b204 is 50, key is test_row_0/B:col10/1733732121801/Put/seqid=0 2024-12-09T08:15:22,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742097_1273 (size=12151) 2024-12-09T08:15:22,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732182871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:22,874 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in d33b04878a7a85043c90681d784a399c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:22,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732182881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:22,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732182882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:22,885 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in d33b04878a7a85043c90681d784a399c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:22,885 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in d33b04878a7a85043c90681d784a399c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:22,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T08:15:22,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:22,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:22,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:22,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:22,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T08:15:23,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T08:15:23,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:23,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:23,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:23,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:23,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:23,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732183151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732183151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:23,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T08:15:23,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:23,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:23,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:23,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:23,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:23,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/9a1f4d334e084fe4b4f70d8e3461b204 2024-12-09T08:15:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T08:15:23,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/1d4b4c54792641569d0a0034b800a134 is 50, key is test_row_0/C:col10/1733732121801/Put/seqid=0 2024-12-09T08:15:23,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742098_1274 (size=12151) 2024-12-09T08:15:23,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T08:15:23,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:23,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:23,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:23,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:23,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:23,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T08:15:23,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:23,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:23,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:23,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:23,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:23,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732183661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:23,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732183661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:23,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/1d4b4c54792641569d0a0034b800a134 2024-12-09T08:15:23,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/089a42e3aa1c44b8bec76eccd30c4148 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/089a42e3aa1c44b8bec76eccd30c4148 2024-12-09T08:15:23,686 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T08:15:23,686 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-09T08:15:23,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/089a42e3aa1c44b8bec76eccd30c4148, entries=200, sequenceid=176, filesize=14.2 K 2024-12-09T08:15:23,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/9a1f4d334e084fe4b4f70d8e3461b204 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/9a1f4d334e084fe4b4f70d8e3461b204 2024-12-09T08:15:23,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/9a1f4d334e084fe4b4f70d8e3461b204, entries=150, sequenceid=176, filesize=11.9 K 2024-12-09T08:15:23,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/1d4b4c54792641569d0a0034b800a134 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/1d4b4c54792641569d0a0034b800a134 2024-12-09T08:15:23,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/1d4b4c54792641569d0a0034b800a134, entries=150, sequenceid=176, filesize=11.9 K 2024-12-09T08:15:23,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for d33b04878a7a85043c90681d784a399c in 1267ms, sequenceid=176, compaction requested=true 2024-12-09T08:15:23,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:23,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:23,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:23,700 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:23,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:23,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:23,700 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:23,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:23,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:23,701 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38967 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:23,701 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/A is initiating minor compaction (all files) 2024-12-09T08:15:23,701 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/A in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:23,701 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/da488ff561d34591b61d4372a2712bd4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c99ab3fe9f404e31a3c0c706fa1c5455, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/089a42e3aa1c44b8bec76eccd30c4148] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=38.1 K 2024-12-09T08:15:23,702 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:23,702 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting da488ff561d34591b61d4372a2712bd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733732118693 2024-12-09T08:15:23,702 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/B is initiating minor compaction (all files) 2024-12-09T08:15:23,702 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/B in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:23,702 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/0e78547564544574ac4654d11b639070, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/1c55e7f9e2d845f5a944835e36f08435, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/9a1f4d334e084fe4b4f70d8e3461b204] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=35.7 K 2024-12-09T08:15:23,702 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c99ab3fe9f404e31a3c0c706fa1c5455, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733732120641 2024-12-09T08:15:23,702 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e78547564544574ac4654d11b639070, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733732118693 2024-12-09T08:15:23,703 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 089a42e3aa1c44b8bec76eccd30c4148, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733732121797 2024-12-09T08:15:23,703 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c55e7f9e2d845f5a944835e36f08435, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733732120641 2024-12-09T08:15:23,703 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a1f4d334e084fe4b4f70d8e3461b204, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733732121797 2024-12-09T08:15:23,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T08:15:23,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:23,712 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-09T08:15:23,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:23,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:23,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:23,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:23,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:23,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:23,723 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:23,723 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#A#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:23,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c2d7b2e169ab4614870ed50e2573886b is 50, key is test_row_0/A:col10/1733732122514/Put/seqid=0 2024-12-09T08:15:23,723 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c6a7410789e94e1fa5c16cff248ba59c is 50, key is test_row_0/A:col10/1733732121801/Put/seqid=0 2024-12-09T08:15:23,723 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/03ed33720a3f4269b1498fa63250ba71 is 50, key is test_row_0/B:col10/1733732121801/Put/seqid=0 2024-12-09T08:15:23,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742099_1275 (size=12527) 2024-12-09T08:15:23,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742101_1277 (size=12527) 2024-12-09T08:15:23,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742100_1276 (size=12151) 2024-12-09T08:15:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T08:15:24,144 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c6a7410789e94e1fa5c16cff248ba59c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c6a7410789e94e1fa5c16cff248ba59c 2024-12-09T08:15:24,148 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c2d7b2e169ab4614870ed50e2573886b 2024-12-09T08:15:24,151 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/03ed33720a3f4269b1498fa63250ba71 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/03ed33720a3f4269b1498fa63250ba71 2024-12-09T08:15:24,152 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/A of d33b04878a7a85043c90681d784a399c into c6a7410789e94e1fa5c16cff248ba59c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:24,152 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:24,153 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/A, priority=13, startTime=1733732123700; duration=0sec 2024-12-09T08:15:24,153 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:24,153 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:A 2024-12-09T08:15:24,153 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:24,156 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:24,156 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/C is initiating minor compaction (all files) 2024-12-09T08:15:24,156 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/C in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:24,156 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/e9055227fca54129a1d3ecd96c73b2e9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/f5edbffd8ebd48e08eb34b46039400b4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/1d4b4c54792641569d0a0034b800a134] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=35.7 K 2024-12-09T08:15:24,157 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9055227fca54129a1d3ecd96c73b2e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733732118693 2024-12-09T08:15:24,157 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5edbffd8ebd48e08eb34b46039400b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733732120641 2024-12-09T08:15:24,158 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d4b4c54792641569d0a0034b800a134, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733732121797 2024-12-09T08:15:24,161 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/B of d33b04878a7a85043c90681d784a399c into 03ed33720a3f4269b1498fa63250ba71(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:24,161 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:24,161 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/B, priority=13, startTime=1733732123700; duration=0sec 2024-12-09T08:15:24,161 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:24,161 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:B 2024-12-09T08:15:24,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/e21ad95fa1cc4a6e88c533faf868814d is 50, key is test_row_0/B:col10/1733732122514/Put/seqid=0 2024-12-09T08:15:24,170 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#C#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:24,170 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/d1b8f228c9f34987a703e41fb47f68b3 is 50, key is test_row_0/C:col10/1733732121801/Put/seqid=0 2024-12-09T08:15:24,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742102_1278 (size=12151) 2024-12-09T08:15:24,178 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/e21ad95fa1cc4a6e88c533faf868814d 2024-12-09T08:15:24,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742103_1279 (size=12527) 2024-12-09T08:15:24,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/a6913be9d4eb4399a355ad29c1274166 is 50, key is test_row_0/C:col10/1733732122514/Put/seqid=0 2024-12-09T08:15:24,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742104_1280 (size=12151) 2024-12-09T08:15:24,584 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/d1b8f228c9f34987a703e41fb47f68b3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/d1b8f228c9f34987a703e41fb47f68b3 2024-12-09T08:15:24,592 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/C of d33b04878a7a85043c90681d784a399c into d1b8f228c9f34987a703e41fb47f68b3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:24,592 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:24,593 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/C, priority=13, startTime=1733732123700; duration=0sec 2024-12-09T08:15:24,593 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:24,593 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:C 2024-12-09T08:15:24,643 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/a6913be9d4eb4399a355ad29c1274166 2024-12-09T08:15:24,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c2d7b2e169ab4614870ed50e2573886b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c2d7b2e169ab4614870ed50e2573886b 2024-12-09T08:15:24,655 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c2d7b2e169ab4614870ed50e2573886b, entries=150, sequenceid=211, filesize=11.9 K 2024-12-09T08:15:24,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/e21ad95fa1cc4a6e88c533faf868814d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e21ad95fa1cc4a6e88c533faf868814d 2024-12-09T08:15:24,676 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e21ad95fa1cc4a6e88c533faf868814d, entries=150, sequenceid=211, filesize=11.9 K 2024-12-09T08:15:24,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/a6913be9d4eb4399a355ad29c1274166 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a6913be9d4eb4399a355ad29c1274166 2024-12-09T08:15:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:24,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:24,685 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a6913be9d4eb4399a355ad29c1274166, entries=150, sequenceid=211, filesize=11.9 K 2024-12-09T08:15:24,687 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=13.42 KB/13740 for d33b04878a7a85043c90681d784a399c in 975ms, sequenceid=211, compaction requested=false 2024-12-09T08:15:24,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:24,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:24,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-09T08:15:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-09T08:15:24,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-09T08:15:24,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0560 sec 2024-12-09T08:15:24,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.0620 sec 2024-12-09T08:15:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:24,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:15:24,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:24,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:24,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:24,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:24,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:24,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:24,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/ed0bcfd0e6bd41df84c13751a7aad08e is 50, key is test_row_0/A:col10/1733732124700/Put/seqid=0 2024-12-09T08:15:24,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742105_1281 (size=14541) 2024-12-09T08:15:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T08:15:24,782 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:24,784 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-09T08:15:24,787 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T08:15:24,788 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:24,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:24,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732124771, max timestamp=1733732124795Num uniques 859; 2024-12-09T08:15:24,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732124771, max timestamp=1733732124795Num uniques 859; 2024-12-09T08:15:24,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732124771, max timestamp=1733732124795Num uniques 859; 2024-12-09T08:15:24,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:24,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:24,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:24,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732184846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:24,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732184851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T08:15:24,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T08:15:24,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:24,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:24,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:24,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:24,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732184961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732184962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:25,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T08:15:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:25,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T08:15:25,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/ed0bcfd0e6bd41df84c13751a7aad08e 2024-12-09T08:15:25,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a2f15a2db9d545d1af809e12c7cccb05 is 50, key is test_row_0/B:col10/1733732124700/Put/seqid=0 2024-12-09T08:15:25,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732185172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:25,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:25,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732185172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:25,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:25,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:25,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:25,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742106_1282 (size=12151) 2024-12-09T08:15:25,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a2f15a2db9d545d1af809e12c7cccb05 2024-12-09T08:15:25,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/702d9200e027424a902fbe2eff4f87b6 is 50, key is test_row_0/C:col10/1733732124700/Put/seqid=0 2024-12-09T08:15:25,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742107_1283 (size=12151) 2024-12-09T08:15:25,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T08:15:25,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:25,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:25,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T08:15:25,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T08:15:25,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:25,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:25,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:25,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:25,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:25,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732185480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:25,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732185481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:25,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T08:15:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:25,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:25,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/702d9200e027424a902fbe2eff4f87b6 2024-12-09T08:15:25,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/ed0bcfd0e6bd41df84c13751a7aad08e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/ed0bcfd0e6bd41df84c13751a7aad08e 2024-12-09T08:15:25,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/ed0bcfd0e6bd41df84c13751a7aad08e, entries=200, sequenceid=225, filesize=14.2 K 2024-12-09T08:15:25,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a2f15a2db9d545d1af809e12c7cccb05 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a2f15a2db9d545d1af809e12c7cccb05 2024-12-09T08:15:25,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a2f15a2db9d545d1af809e12c7cccb05, entries=150, sequenceid=225, filesize=11.9 K 2024-12-09T08:15:25,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/702d9200e027424a902fbe2eff4f87b6 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/702d9200e027424a902fbe2eff4f87b6 2024-12-09T08:15:25,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/702d9200e027424a902fbe2eff4f87b6, entries=150, sequenceid=225, filesize=11.9 K 2024-12-09T08:15:25,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d33b04878a7a85043c90681d784a399c in 971ms, sequenceid=225, compaction requested=true 2024-12-09T08:15:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:25,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:25,673 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:25,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:25,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:25,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:25,673 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:25,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:25,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:25,674 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:25,674 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:25,674 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/B is initiating minor compaction (all files) 2024-12-09T08:15:25,674 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/A is initiating minor compaction (all files) 2024-12-09T08:15:25,674 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/B in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:25,674 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/A in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:25,674 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/03ed33720a3f4269b1498fa63250ba71, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e21ad95fa1cc4a6e88c533faf868814d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a2f15a2db9d545d1af809e12c7cccb05] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=36.0 K 2024-12-09T08:15:25,674 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c6a7410789e94e1fa5c16cff248ba59c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c2d7b2e169ab4614870ed50e2573886b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/ed0bcfd0e6bd41df84c13751a7aad08e] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=38.3 K 2024-12-09T08:15:25,674 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6a7410789e94e1fa5c16cff248ba59c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733732121797 2024-12-09T08:15:25,674 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ed33720a3f4269b1498fa63250ba71, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733732121797 2024-12-09T08:15:25,675 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2d7b2e169ab4614870ed50e2573886b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733732122514 2024-12-09T08:15:25,675 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting e21ad95fa1cc4a6e88c533faf868814d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733732122514 2024-12-09T08:15:25,675 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f15a2db9d545d1af809e12c7cccb05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733732124693 2024-12-09T08:15:25,675 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed0bcfd0e6bd41df84c13751a7aad08e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733732124676 2024-12-09T08:15:25,680 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#A#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:25,681 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/4d9038862bc945c0a43747fffcef7f39 is 50, key is test_row_0/A:col10/1733732124700/Put/seqid=0 2024-12-09T08:15:25,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742108_1284 (size=12629) 2024-12-09T08:15:25,690 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#B#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:25,691 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/ba1e5736467544beb9c131735f086164 is 50, key is test_row_0/B:col10/1733732124700/Put/seqid=0 2024-12-09T08:15:25,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742109_1285 (size=12629) 2024-12-09T08:15:25,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T08:15:25,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:25,717 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-09T08:15:25,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:25,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:25,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:25,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:25,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:25,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:25,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/9db5c105f16c42479f7bdadc57697d1b is 50, key is test_row_0/A:col10/1733732124833/Put/seqid=0 2024-12-09T08:15:25,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742110_1286 (size=12201) 2024-12-09T08:15:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T08:15:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:25,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:26,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732186004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:26,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732186006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:26,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:26,088 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/4d9038862bc945c0a43747fffcef7f39 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/4d9038862bc945c0a43747fffcef7f39 2024-12-09T08:15:26,092 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/A of d33b04878a7a85043c90681d784a399c into 4d9038862bc945c0a43747fffcef7f39(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:26,093 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:26,093 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/A, priority=13, startTime=1733732125672; duration=0sec 2024-12-09T08:15:26,093 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:26,093 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:A 2024-12-09T08:15:26,093 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:26,094 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:26,094 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/C is initiating minor compaction (all files) 2024-12-09T08:15:26,094 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/C in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:26,094 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/d1b8f228c9f34987a703e41fb47f68b3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a6913be9d4eb4399a355ad29c1274166, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/702d9200e027424a902fbe2eff4f87b6] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=36.0 K 2024-12-09T08:15:26,094 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b8f228c9f34987a703e41fb47f68b3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733732121797 2024-12-09T08:15:26,094 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6913be9d4eb4399a355ad29c1274166, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733732122514 2024-12-09T08:15:26,095 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702d9200e027424a902fbe2eff4f87b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733732124693 2024-12-09T08:15:26,100 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/ba1e5736467544beb9c131735f086164 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/ba1e5736467544beb9c131735f086164 2024-12-09T08:15:26,101 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#C#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:26,101 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/8c604ae8cb5546d8b47dbfff48b798e4 is 50, key is test_row_0/C:col10/1733732124700/Put/seqid=0 2024-12-09T08:15:26,105 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/B of d33b04878a7a85043c90681d784a399c into ba1e5736467544beb9c131735f086164(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:26,105 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:26,105 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/B, priority=13, startTime=1733732125673; duration=0sec 2024-12-09T08:15:26,106 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:26,106 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:B 2024-12-09T08:15:26,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742111_1287 (size=12629) 2024-12-09T08:15:26,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732186111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:26,114 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/8c604ae8cb5546d8b47dbfff48b798e4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/8c604ae8cb5546d8b47dbfff48b798e4 2024-12-09T08:15:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:26,118 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/C of d33b04878a7a85043c90681d784a399c into 8c604ae8cb5546d8b47dbfff48b798e4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:26,118 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:26,118 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/C, priority=13, startTime=1733732125673; duration=0sec 2024-12-09T08:15:26,118 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:26,119 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:C 2024-12-09T08:15:26,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732186121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:26,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:26,124 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/9db5c105f16c42479f7bdadc57697d1b 2024-12-09T08:15:26,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/eba030051ef645f3bba69b62bbf7fd13 is 50, key is test_row_0/B:col10/1733732124833/Put/seqid=0 2024-12-09T08:15:26,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742112_1288 (size=12201) 2024-12-09T08:15:26,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732186321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:26,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:26,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732186331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:26,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:26,536 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/eba030051ef645f3bba69b62bbf7fd13 2024-12-09T08:15:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/5beae54798704d3596ca62d1c70bccb0 is 50, key is test_row_0/C:col10/1733732124833/Put/seqid=0 2024-12-09T08:15:26,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742113_1289 (size=12201) 2024-12-09T08:15:26,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732186631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:26,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732186641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:26,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:26,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35558 deadline: 1733732186891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:26,893 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in d33b04878a7a85043c90681d784a399c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:26,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35616 deadline: 1733732186911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:26,913 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in d33b04878a7a85043c90681d784a399c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:26,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:26,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35572 deadline: 1733732186921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:26,923 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in d33b04878a7a85043c90681d784a399c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T08:15:26,947 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/5beae54798704d3596ca62d1c70bccb0 2024-12-09T08:15:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/9db5c105f16c42479f7bdadc57697d1b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/9db5c105f16c42479f7bdadc57697d1b 2024-12-09T08:15:26,957 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/9db5c105f16c42479f7bdadc57697d1b, entries=150, sequenceid=261, filesize=11.9 K 2024-12-09T08:15:26,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/eba030051ef645f3bba69b62bbf7fd13 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/eba030051ef645f3bba69b62bbf7fd13 2024-12-09T08:15:26,963 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/eba030051ef645f3bba69b62bbf7fd13, entries=150, sequenceid=261, filesize=11.9 K 2024-12-09T08:15:26,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/5beae54798704d3596ca62d1c70bccb0 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5beae54798704d3596ca62d1c70bccb0 2024-12-09T08:15:26,967 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5beae54798704d3596ca62d1c70bccb0, entries=150, sequenceid=261, filesize=11.9 K 2024-12-09T08:15:26,968 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for d33b04878a7a85043c90681d784a399c in 1251ms, sequenceid=261, compaction requested=false 2024-12-09T08:15:26,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:26,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:26,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-09T08:15:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-09T08:15:26,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-09T08:15:26,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1810 sec 2024-12-09T08:15:26,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.1870 sec 2024-12-09T08:15:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:27,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T08:15:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:27,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/3f7d9ad6a82b4ce686e2000540feefda is 50, key is test_row_0/A:col10/1733732125999/Put/seqid=0 2024-12-09T08:15:27,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742114_1290 (size=14741) 2024-12-09T08:15:27,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732187201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:27,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732187203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:27,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732187311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:27,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732187312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:27,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732187521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732187521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:27,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/3f7d9ad6a82b4ce686e2000540feefda 2024-12-09T08:15:27,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/d83cd970fca9492db1c202fe7c1865ec is 50, key is test_row_0/B:col10/1733732125999/Put/seqid=0 2024-12-09T08:15:27,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742115_1291 (size=12301) 2024-12-09T08:15:27,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732187831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:27,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732187831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:27,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/d83cd970fca9492db1c202fe7c1865ec 2024-12-09T08:15:28,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/5d574b21a3c24143ab09b77db56dd53f is 50, key is test_row_0/C:col10/1733732125999/Put/seqid=0 2024-12-09T08:15:28,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742116_1292 (size=12301) 2024-12-09T08:15:28,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732188341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:28,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732188341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:28,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/5d574b21a3c24143ab09b77db56dd53f 2024-12-09T08:15:28,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/3f7d9ad6a82b4ce686e2000540feefda as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/3f7d9ad6a82b4ce686e2000540feefda 2024-12-09T08:15:28,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/3f7d9ad6a82b4ce686e2000540feefda, entries=200, sequenceid=276, filesize=14.4 K 2024-12-09T08:15:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/d83cd970fca9492db1c202fe7c1865ec as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/d83cd970fca9492db1c202fe7c1865ec 2024-12-09T08:15:28,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/d83cd970fca9492db1c202fe7c1865ec, entries=150, sequenceid=276, filesize=12.0 K 2024-12-09T08:15:28,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/5d574b21a3c24143ab09b77db56dd53f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5d574b21a3c24143ab09b77db56dd53f 2024-12-09T08:15:28,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5d574b21a3c24143ab09b77db56dd53f, entries=150, sequenceid=276, filesize=12.0 K 2024-12-09T08:15:28,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d33b04878a7a85043c90681d784a399c in 1282ms, sequenceid=276, compaction requested=true 2024-12-09T08:15:28,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:28,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:28,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:28,425 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:28,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:28,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:28,425 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:28,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:28,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:28,426 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37131 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:28,426 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:28,426 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/A is initiating minor compaction (all files) 2024-12-09T08:15:28,426 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/B is initiating minor compaction (all files) 2024-12-09T08:15:28,426 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/A in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:28,426 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/B in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:28,426 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/4d9038862bc945c0a43747fffcef7f39, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/9db5c105f16c42479f7bdadc57697d1b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/3f7d9ad6a82b4ce686e2000540feefda] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=38.6 K 2024-12-09T08:15:28,426 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/ba1e5736467544beb9c131735f086164, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/eba030051ef645f3bba69b62bbf7fd13, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/d83cd970fca9492db1c202fe7c1865ec] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=36.3 K 2024-12-09T08:15:28,426 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d9038862bc945c0a43747fffcef7f39, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733732124693 2024-12-09T08:15:28,426 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ba1e5736467544beb9c131735f086164, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733732124693 2024-12-09T08:15:28,426 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting eba030051ef645f3bba69b62bbf7fd13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733732124833 2024-12-09T08:15:28,427 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9db5c105f16c42479f7bdadc57697d1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733732124833 2024-12-09T08:15:28,427 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting d83cd970fca9492db1c202fe7c1865ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733732125999 2024-12-09T08:15:28,427 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f7d9ad6a82b4ce686e2000540feefda, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733732125999 2024-12-09T08:15:28,433 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#B#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:28,433 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#A#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:28,433 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/48a281d7b6a446949388b708c4f211e0 is 50, key is test_row_0/B:col10/1733732125999/Put/seqid=0 2024-12-09T08:15:28,433 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/98225ab92a2d4fadbac04531abc9d9fb is 50, key is test_row_0/A:col10/1733732125999/Put/seqid=0 2024-12-09T08:15:28,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742117_1293 (size=12881) 2024-12-09T08:15:28,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742118_1294 (size=12881) 2024-12-09T08:15:28,845 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/98225ab92a2d4fadbac04531abc9d9fb as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/98225ab92a2d4fadbac04531abc9d9fb 2024-12-09T08:15:28,845 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/48a281d7b6a446949388b708c4f211e0 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/48a281d7b6a446949388b708c4f211e0 2024-12-09T08:15:28,856 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/B of d33b04878a7a85043c90681d784a399c into 48a281d7b6a446949388b708c4f211e0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:28,856 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:28,856 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/B, priority=13, startTime=1733732128425; duration=0sec 2024-12-09T08:15:28,856 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/A of d33b04878a7a85043c90681d784a399c into 98225ab92a2d4fadbac04531abc9d9fb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:28,856 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:28,856 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:28,856 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:B 2024-12-09T08:15:28,856 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/A, priority=13, startTime=1733732128425; duration=0sec 2024-12-09T08:15:28,856 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:28,856 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:28,857 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:A 2024-12-09T08:15:28,858 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37131 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:28,858 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/C is initiating minor compaction (all files) 2024-12-09T08:15:28,858 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/C in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:28,858 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/8c604ae8cb5546d8b47dbfff48b798e4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5beae54798704d3596ca62d1c70bccb0, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5d574b21a3c24143ab09b77db56dd53f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=36.3 K 2024-12-09T08:15:28,859 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c604ae8cb5546d8b47dbfff48b798e4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733732124693 2024-12-09T08:15:28,859 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5beae54798704d3596ca62d1c70bccb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733732124833 2024-12-09T08:15:28,859 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d574b21a3c24143ab09b77db56dd53f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733732125999 2024-12-09T08:15:28,864 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:28,865 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/5dd24cf5cd634e71bdcced03f43bc6e9 is 50, key is test_row_0/C:col10/1733732125999/Put/seqid=0 2024-12-09T08:15:28,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742119_1295 (size=12881) 2024-12-09T08:15:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T08:15:28,951 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:28,951 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-09T08:15:28,953 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T08:15:28,954 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:28,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T08:15:29,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T08:15:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:29,108 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T08:15:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/e13ee9d7f2d542b0bf59ef9441af133b is 50, key is test_row_0/A:col10/1733732127198/Put/seqid=0 2024-12-09T08:15:29,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742120_1296 (size=12301) 2024-12-09T08:15:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T08:15:29,299 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/5dd24cf5cd634e71bdcced03f43bc6e9 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5dd24cf5cd634e71bdcced03f43bc6e9 2024-12-09T08:15:29,306 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/C of d33b04878a7a85043c90681d784a399c into 5dd24cf5cd634e71bdcced03f43bc6e9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:29,306 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:29,306 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/C, priority=13, startTime=1733732128425; duration=0sec 2024-12-09T08:15:29,306 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:29,306 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:C 2024-12-09T08:15:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:29,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:29,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732189375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:29,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:29,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:29,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:29,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732189376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:29,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:29,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:29,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:29,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732189481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:29,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732189490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:29,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:29,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:29,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:29,520 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/e13ee9d7f2d542b0bf59ef9441af133b 2024-12-09T08:15:29,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a9e1e6bca3274b3fb558cb3ac8cd379e is 50, key is test_row_0/B:col10/1733732127198/Put/seqid=0 2024-12-09T08:15:29,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742121_1297 (size=12301) 2024-12-09T08:15:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T08:15:29,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732189691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:29,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732189701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:29,935 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a9e1e6bca3274b3fb558cb3ac8cd379e 2024-12-09T08:15:29,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/944bd484228d41f29647506ade1abbdc is 50, key is test_row_0/C:col10/1733732127198/Put/seqid=0 2024-12-09T08:15:29,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742122_1298 (size=12301) 2024-12-09T08:15:29,947 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/944bd484228d41f29647506ade1abbdc 2024-12-09T08:15:29,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/e13ee9d7f2d542b0bf59ef9441af133b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/e13ee9d7f2d542b0bf59ef9441af133b 2024-12-09T08:15:29,956 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/e13ee9d7f2d542b0bf59ef9441af133b, entries=150, sequenceid=298, filesize=12.0 K 2024-12-09T08:15:29,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a9e1e6bca3274b3fb558cb3ac8cd379e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a9e1e6bca3274b3fb558cb3ac8cd379e 2024-12-09T08:15:29,962 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a9e1e6bca3274b3fb558cb3ac8cd379e, entries=150, sequenceid=298, filesize=12.0 K 2024-12-09T08:15:29,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/944bd484228d41f29647506ade1abbdc as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/944bd484228d41f29647506ade1abbdc 2024-12-09T08:15:29,972 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/944bd484228d41f29647506ade1abbdc, entries=150, sequenceid=298, filesize=12.0 K 2024-12-09T08:15:29,973 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d33b04878a7a85043c90681d784a399c in 865ms, sequenceid=298, compaction requested=false 2024-12-09T08:15:29,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:29,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:29,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-09T08:15:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-09T08:15:29,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-09T08:15:29,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0220 sec 2024-12-09T08:15:29,980 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0280 sec 2024-12-09T08:15:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:30,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T08:15:30,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:30,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:30,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:30,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:30,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:30,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:30,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c68e704638d04b35a88eecc1e8f5e4fa is 50, key is test_row_0/A:col10/1733732129374/Put/seqid=0 2024-12-09T08:15:30,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742123_1299 (size=12301) 2024-12-09T08:15:30,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732190051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:30,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:30,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732190052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T08:15:30,091 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:30,092 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-09T08:15:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T08:15:30,098 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:30,099 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:30,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:30,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732190160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732190161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T08:15:30,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T08:15:30,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:30,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:30,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:30,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732190371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:30,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732190371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:30,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T08:15:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:30,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T08:15:30,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c68e704638d04b35a88eecc1e8f5e4fa 2024-12-09T08:15:30,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/10c6fafa6ef1468a8b6ad6df6ca0ed0f is 50, key is test_row_0/B:col10/1733732129374/Put/seqid=0 2024-12-09T08:15:30,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742124_1300 (size=12301) 2024-12-09T08:15:30,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T08:15:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:30,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732190681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:30,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732190681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:30,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T08:15:30,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:30,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:30,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T08:15:30,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/10c6fafa6ef1468a8b6ad6df6ca0ed0f 2024-12-09T08:15:30,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/6e20afa89a924085b4336539e46b2f3c is 50, key is test_row_0/C:col10/1733732129374/Put/seqid=0 2024-12-09T08:15:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742125_1301 (size=12301) 2024-12-09T08:15:30,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T08:15:30,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:30,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:30,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:30,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:31,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T08:15:31,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:31,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:31,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:31,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:31,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:31,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T08:15:31,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:31,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:31,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:31,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:31,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732191191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732191191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:31,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/6e20afa89a924085b4336539e46b2f3c 2024-12-09T08:15:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T08:15:31,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c68e704638d04b35a88eecc1e8f5e4fa as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c68e704638d04b35a88eecc1e8f5e4fa 2024-12-09T08:15:31,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c68e704638d04b35a88eecc1e8f5e4fa, entries=150, sequenceid=312, filesize=12.0 K 2024-12-09T08:15:31,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/10c6fafa6ef1468a8b6ad6df6ca0ed0f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/10c6fafa6ef1468a8b6ad6df6ca0ed0f 2024-12-09T08:15:31,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/10c6fafa6ef1468a8b6ad6df6ca0ed0f, entries=150, sequenceid=312, filesize=12.0 K 2024-12-09T08:15:31,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/6e20afa89a924085b4336539e46b2f3c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/6e20afa89a924085b4336539e46b2f3c 2024-12-09T08:15:31,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/6e20afa89a924085b4336539e46b2f3c, entries=150, sequenceid=312, filesize=12.0 K 2024-12-09T08:15:31,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d33b04878a7a85043c90681d784a399c in 1253ms, sequenceid=312, compaction requested=true 2024-12-09T08:15:31,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:31,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:31,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:31,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:31,256 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:31,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:31,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:31,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:31,256 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:31,257 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:31,257 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/B is initiating minor compaction (all files) 2024-12-09T08:15:31,257 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/B in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:31,258 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/48a281d7b6a446949388b708c4f211e0, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a9e1e6bca3274b3fb558cb3ac8cd379e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/10c6fafa6ef1468a8b6ad6df6ca0ed0f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=36.6 K 2024-12-09T08:15:31,258 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:31,258 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/A is initiating minor compaction (all files) 2024-12-09T08:15:31,258 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/A in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:31,258 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/98225ab92a2d4fadbac04531abc9d9fb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/e13ee9d7f2d542b0bf59ef9441af133b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c68e704638d04b35a88eecc1e8f5e4fa] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=36.6 K 2024-12-09T08:15:31,259 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98225ab92a2d4fadbac04531abc9d9fb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733732125999 2024-12-09T08:15:31,259 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 48a281d7b6a446949388b708c4f211e0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733732125999 2024-12-09T08:15:31,260 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e13ee9d7f2d542b0bf59ef9441af133b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733732127198 2024-12-09T08:15:31,260 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a9e1e6bca3274b3fb558cb3ac8cd379e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733732127198 2024-12-09T08:15:31,261 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c68e704638d04b35a88eecc1e8f5e4fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733732129374 2024-12-09T08:15:31,261 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 10c6fafa6ef1468a8b6ad6df6ca0ed0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733732129374 2024-12-09T08:15:31,271 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#B#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:31,271 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/26d45d8dfbbd42ba8ec7c33929bf53be is 50, key is test_row_0/B:col10/1733732129374/Put/seqid=0 2024-12-09T08:15:31,273 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#A#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:31,274 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/7c7f9d9bf99c43fcbab91529c48d1af3 is 50, key is test_row_0/A:col10/1733732129374/Put/seqid=0 2024-12-09T08:15:31,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742126_1302 (size=12983) 2024-12-09T08:15:31,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742127_1303 (size=12983) 2024-12-09T08:15:31,311 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/7c7f9d9bf99c43fcbab91529c48d1af3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/7c7f9d9bf99c43fcbab91529c48d1af3 2024-12-09T08:15:31,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T08:15:31,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:31,320 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T08:15:31,321 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/A of d33b04878a7a85043c90681d784a399c into 7c7f9d9bf99c43fcbab91529c48d1af3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:31,321 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:31,321 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/A, priority=13, startTime=1733732131256; duration=0sec 2024-12-09T08:15:31,321 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:31,321 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:A 2024-12-09T08:15:31,321 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:31,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:31,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:31,324 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:31,324 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/C is initiating minor compaction (all files) 2024-12-09T08:15:31,324 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/C in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:31,324 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5dd24cf5cd634e71bdcced03f43bc6e9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/944bd484228d41f29647506ade1abbdc, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/6e20afa89a924085b4336539e46b2f3c] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=36.6 K 2024-12-09T08:15:31,325 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dd24cf5cd634e71bdcced03f43bc6e9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733732125999 2024-12-09T08:15:31,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/edaaba0f32474257a729557fe10e3a50 is 50, key is test_row_0/A:col10/1733732130049/Put/seqid=0 2024-12-09T08:15:31,327 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 944bd484228d41f29647506ade1abbdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733732127198 2024-12-09T08:15:31,328 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e20afa89a924085b4336539e46b2f3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733732129374 2024-12-09T08:15:31,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742128_1304 (size=12301) 2024-12-09T08:15:31,334 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:31,334 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/04e76b3fa44d4dcfb4cdcc9250de9b18 is 50, key is test_row_0/C:col10/1733732129374/Put/seqid=0 2024-12-09T08:15:31,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742129_1305 (size=12983) 2024-12-09T08:15:31,704 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/26d45d8dfbbd42ba8ec7c33929bf53be as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/26d45d8dfbbd42ba8ec7c33929bf53be 2024-12-09T08:15:31,707 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/B of d33b04878a7a85043c90681d784a399c into 26d45d8dfbbd42ba8ec7c33929bf53be(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:31,708 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:31,708 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/B, priority=13, startTime=1733732131256; duration=0sec 2024-12-09T08:15:31,708 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:31,708 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:B 2024-12-09T08:15:31,732 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/edaaba0f32474257a729557fe10e3a50 2024-12-09T08:15:31,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a3b31def2f784f9cb2716175fbf24c60 is 50, key is test_row_0/B:col10/1733732130049/Put/seqid=0 2024-12-09T08:15:31,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742130_1306 (size=12301) 2024-12-09T08:15:31,769 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/04e76b3fa44d4dcfb4cdcc9250de9b18 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/04e76b3fa44d4dcfb4cdcc9250de9b18 2024-12-09T08:15:31,778 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/C of d33b04878a7a85043c90681d784a399c into 04e76b3fa44d4dcfb4cdcc9250de9b18(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:31,778 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:31,778 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/C, priority=13, startTime=1733732131256; duration=0sec 2024-12-09T08:15:31,778 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:31,778 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:C 2024-12-09T08:15:32,157 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a3b31def2f784f9cb2716175fbf24c60 2024-12-09T08:15:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/a16a9b8f4cc04eb58aa9362c57e8fd15 is 50, key is test_row_0/C:col10/1733732130049/Put/seqid=0 2024-12-09T08:15:32,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742131_1307 (size=12301) 2024-12-09T08:15:32,170 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/a16a9b8f4cc04eb58aa9362c57e8fd15 2024-12-09T08:15:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/edaaba0f32474257a729557fe10e3a50 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/edaaba0f32474257a729557fe10e3a50 2024-12-09T08:15:32,183 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/edaaba0f32474257a729557fe10e3a50, entries=150, sequenceid=332, filesize=12.0 K 2024-12-09T08:15:32,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a3b31def2f784f9cb2716175fbf24c60 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a3b31def2f784f9cb2716175fbf24c60 2024-12-09T08:15:32,196 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a3b31def2f784f9cb2716175fbf24c60, entries=150, sequenceid=332, filesize=12.0 K 2024-12-09T08:15:32,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/a16a9b8f4cc04eb58aa9362c57e8fd15 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a16a9b8f4cc04eb58aa9362c57e8fd15 2024-12-09T08:15:32,201 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a16a9b8f4cc04eb58aa9362c57e8fd15, entries=150, sequenceid=332, filesize=12.0 K 2024-12-09T08:15:32,202 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for d33b04878a7a85043c90681d784a399c in 882ms, sequenceid=332, compaction requested=false 2024-12-09T08:15:32,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:32,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:32,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-09T08:15:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-09T08:15:32,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-09T08:15:32,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1050 sec 2024-12-09T08:15:32,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.1160 sec 2024-12-09T08:15:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:32,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:15:32,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:32,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:32,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:32,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:32,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:32,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:32,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/ded50c40ab5a4b8aa98d84fdb52d344d is 50, key is test_row_0/A:col10/1733732132218/Put/seqid=0 2024-12-09T08:15:32,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742132_1308 (size=12301) 2024-12-09T08:15:32,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/ded50c40ab5a4b8aa98d84fdb52d344d 2024-12-09T08:15:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T08:15:32,241 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:32,241 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-09T08:15:32,243 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:32,244 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:32,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T08:15:32,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/19286c57bd1d4494ba3f16f6238cb10b is 50, key is test_row_0/B:col10/1733732132218/Put/seqid=0 2024-12-09T08:15:32,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742133_1309 (size=12301) 2024-12-09T08:15:32,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/19286c57bd1d4494ba3f16f6238cb10b 2024-12-09T08:15:32,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/0642335f68664043a244c0219ad87bd4 is 50, key is test_row_0/C:col10/1733732132218/Put/seqid=0 2024-12-09T08:15:32,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732132245, max timestamp=1733732132264Num uniques 859; 2024-12-09T08:15:32,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:32,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742134_1310 (size=12301) 2024-12-09T08:15:32,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732132245, max timestamp=1733732132264Num uniques 859; 2024-12-09T08:15:32,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732132245, max timestamp=1733732132264Num uniques 859; 2024-12-09T08:15:32,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:32,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:32,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/0642335f68664043a244c0219ad87bd4 2024-12-09T08:15:32,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/ded50c40ab5a4b8aa98d84fdb52d344d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/ded50c40ab5a4b8aa98d84fdb52d344d 2024-12-09T08:15:32,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/ded50c40ab5a4b8aa98d84fdb52d344d, entries=150, sequenceid=345, filesize=12.0 K 2024-12-09T08:15:32,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/19286c57bd1d4494ba3f16f6238cb10b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/19286c57bd1d4494ba3f16f6238cb10b 2024-12-09T08:15:32,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/19286c57bd1d4494ba3f16f6238cb10b, entries=150, sequenceid=345, filesize=12.0 K 2024-12-09T08:15:32,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/0642335f68664043a244c0219ad87bd4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/0642335f68664043a244c0219ad87bd4 2024-12-09T08:15:32,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/0642335f68664043a244c0219ad87bd4, entries=150, sequenceid=345, filesize=12.0 K 2024-12-09T08:15:32,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=107.34 KB/109920 for d33b04878a7a85043c90681d784a399c in 101ms, sequenceid=345, compaction requested=true 2024-12-09T08:15:32,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:32,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:32,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:32,321 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:32,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:32,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:32,321 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:32,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:32,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:32,322 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:32,322 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:32,322 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/B is initiating minor compaction (all files) 2024-12-09T08:15:32,322 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/A is initiating minor compaction (all files) 2024-12-09T08:15:32,322 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/A in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:32,322 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/B in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:32,322 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/26d45d8dfbbd42ba8ec7c33929bf53be, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a3b31def2f784f9cb2716175fbf24c60, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/19286c57bd1d4494ba3f16f6238cb10b] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=36.7 K 2024-12-09T08:15:32,322 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/7c7f9d9bf99c43fcbab91529c48d1af3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/edaaba0f32474257a729557fe10e3a50, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/ded50c40ab5a4b8aa98d84fdb52d344d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=36.7 K 2024-12-09T08:15:32,322 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d45d8dfbbd42ba8ec7c33929bf53be, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733732129374 2024-12-09T08:15:32,322 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c7f9d9bf99c43fcbab91529c48d1af3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733732129374 2024-12-09T08:15:32,322 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b31def2f784f9cb2716175fbf24c60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733732130043 2024-12-09T08:15:32,323 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting edaaba0f32474257a729557fe10e3a50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733732130043 2024-12-09T08:15:32,323 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 19286c57bd1d4494ba3f16f6238cb10b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733732132214 2024-12-09T08:15:32,323 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ded50c40ab5a4b8aa98d84fdb52d344d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733732132214 2024-12-09T08:15:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:32,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=114.05 KB heapSize=325.22 KB 2024-12-09T08:15:32,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:32,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:32,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:32,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:32,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:32,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:32,347 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#A#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:32,347 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/7b6e1227e364465f91cc779cb1059890 is 50, key is test_row_0/A:col10/1733732132218/Put/seqid=0 2024-12-09T08:15:32,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/1efa76881a1f4c9ca3051a3ff3cd274a is 50, key is test_row_0/A:col10/1733732132319/Put/seqid=0 2024-12-09T08:15:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T08:15:32,353 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:32,353 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/09fe20b69013423ca43c0eb55e7f0450 is 50, key is test_row_0/B:col10/1733732132218/Put/seqid=0 2024-12-09T08:15:32,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732192365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:32,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:32,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:32,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:32,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732192370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:32,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:32,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:32,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:32,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742135_1311 (size=14741) 2024-12-09T08:15:32,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T08:15:32,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:32,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:32,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:32,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:32,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:32,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742136_1312 (size=13085) 2024-12-09T08:15:32,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742137_1313 (size=13085) 2024-12-09T08:15:32,453 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/7b6e1227e364465f91cc779cb1059890 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/7b6e1227e364465f91cc779cb1059890 2024-12-09T08:15:32,464 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/A of d33b04878a7a85043c90681d784a399c into 7b6e1227e364465f91cc779cb1059890(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:32,464 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:32,465 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/A, priority=13, startTime=1733732132321; duration=0sec 2024-12-09T08:15:32,465 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:32,465 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:A 2024-12-09T08:15:32,465 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:32,467 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:32,467 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/C is initiating minor compaction (all files) 2024-12-09T08:15:32,467 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/C in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:32,467 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/04e76b3fa44d4dcfb4cdcc9250de9b18, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a16a9b8f4cc04eb58aa9362c57e8fd15, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/0642335f68664043a244c0219ad87bd4] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=36.7 K 2024-12-09T08:15:32,467 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04e76b3fa44d4dcfb4cdcc9250de9b18, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733732129374 2024-12-09T08:15:32,468 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a16a9b8f4cc04eb58aa9362c57e8fd15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733732130043 2024-12-09T08:15:32,468 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0642335f68664043a244c0219ad87bd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733732132214 2024-12-09T08:15:32,483 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:32,483 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/ffe3f71941b74518beb91578348a9abe is 50, key is test_row_0/C:col10/1733732132218/Put/seqid=0 2024-12-09T08:15:32,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:32,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:32,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732192485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:32,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732192485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:32,511 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T08:15:32,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742138_1314 (size=13085) 2024-12-09T08:15:32,535 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/ffe3f71941b74518beb91578348a9abe as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/ffe3f71941b74518beb91578348a9abe 2024-12-09T08:15:32,551 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/C of d33b04878a7a85043c90681d784a399c into ffe3f71941b74518beb91578348a9abe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:32,552 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:32,552 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/C, priority=13, startTime=1733732132321; duration=0sec 2024-12-09T08:15:32,552 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:32,552 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:C 2024-12-09T08:15:32,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T08:15:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:32,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T08:15:32,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732192702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:32,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732192702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:32,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T08:15:32,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:32,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:32,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:32,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:32,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:32,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/1efa76881a1f4c9ca3051a3ff3cd274a 2024-12-09T08:15:32,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a6aab1cb1a89499385262690f52ed9a3 is 50, key is test_row_0/B:col10/1733732132319/Put/seqid=0 2024-12-09T08:15:32,832 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/09fe20b69013423ca43c0eb55e7f0450 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/09fe20b69013423ca43c0eb55e7f0450 2024-12-09T08:15:32,847 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d33b04878a7a85043c90681d784a399c/B of d33b04878a7a85043c90681d784a399c into 09fe20b69013423ca43c0eb55e7f0450(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:32,847 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:32,847 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/B, priority=13, startTime=1733732132321; duration=0sec 2024-12-09T08:15:32,847 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:32,847 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:B 2024-12-09T08:15:32,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742139_1315 (size=12301) 2024-12-09T08:15:32,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a6aab1cb1a89499385262690f52ed9a3 2024-12-09T08:15:32,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T08:15:32,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:32,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:32,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:32,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:32,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T08:15:32,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/cf211007703e49559e219b5325b082d5 is 50, key is test_row_0/C:col10/1733732132319/Put/seqid=0 2024-12-09T08:15:32,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742140_1316 (size=12301) 2024-12-09T08:15:33,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732193011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:33,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732193011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:33,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T08:15:33,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:33,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:33,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:33,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:33,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:33,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T08:15:33,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:33,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:33,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:33,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:33,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:33,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:33,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/cf211007703e49559e219b5325b082d5 2024-12-09T08:15:33,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/1efa76881a1f4c9ca3051a3ff3cd274a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/1efa76881a1f4c9ca3051a3ff3cd274a 2024-12-09T08:15:33,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/1efa76881a1f4c9ca3051a3ff3cd274a, entries=200, sequenceid=380, filesize=14.4 K 2024-12-09T08:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/a6aab1cb1a89499385262690f52ed9a3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a6aab1cb1a89499385262690f52ed9a3 2024-12-09T08:15:33,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a6aab1cb1a89499385262690f52ed9a3, entries=150, sequenceid=380, filesize=12.0 K 2024-12-09T08:15:33,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/cf211007703e49559e219b5325b082d5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/cf211007703e49559e219b5325b082d5 2024-12-09T08:15:33,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/cf211007703e49559e219b5325b082d5, entries=150, sequenceid=380, filesize=12.0 K 2024-12-09T08:15:33,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for d33b04878a7a85043c90681d784a399c in 995ms, sequenceid=380, compaction requested=false 2024-12-09T08:15:33,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:33,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T08:15:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:33,324 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T08:15:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:33,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/79cfc4d6e9a74c8e8f1044a655adcef2 is 50, key is test_row_0/A:col10/1733732132363/Put/seqid=0 2024-12-09T08:15:33,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742141_1317 (size=12301) 2024-12-09T08:15:33,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T08:15:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:33,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. as already flushing 2024-12-09T08:15:33,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732193557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:33,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732193562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:33,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732193671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732193671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:33,760 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/79cfc4d6e9a74c8e8f1044a655adcef2 2024-12-09T08:15:33,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/51908211018c4ec2aad3d183667d303c is 50, key is test_row_0/B:col10/1733732132363/Put/seqid=0 2024-12-09T08:15:33,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742142_1318 (size=12301) 2024-12-09T08:15:33,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732193881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732193881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:34,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:34,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:34,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1733732194191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:34,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35612 deadline: 1733732194191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:34,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:34,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:34,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:34,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:34,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:34,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:15:34,198 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/51908211018c4ec2aad3d183667d303c 2024-12-09T08:15:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/e654ae0f183d4a80810b5772e2e68fd2 is 50, key is test_row_0/C:col10/1733732132363/Put/seqid=0 2024-12-09T08:15:34,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742143_1319 (size=12301) 2024-12-09T08:15:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T08:15:34,608 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/e654ae0f183d4a80810b5772e2e68fd2 2024-12-09T08:15:34,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/79cfc4d6e9a74c8e8f1044a655adcef2 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/79cfc4d6e9a74c8e8f1044a655adcef2 2024-12-09T08:15:34,616 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/79cfc4d6e9a74c8e8f1044a655adcef2, entries=150, sequenceid=395, filesize=12.0 K 2024-12-09T08:15:34,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/51908211018c4ec2aad3d183667d303c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/51908211018c4ec2aad3d183667d303c 2024-12-09T08:15:34,623 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/51908211018c4ec2aad3d183667d303c, entries=150, sequenceid=395, filesize=12.0 K 2024-12-09T08:15:34,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/e654ae0f183d4a80810b5772e2e68fd2 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/e654ae0f183d4a80810b5772e2e68fd2 2024-12-09T08:15:34,627 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/e654ae0f183d4a80810b5772e2e68fd2, entries=150, sequenceid=395, filesize=12.0 K 2024-12-09T08:15:34,628 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d33b04878a7a85043c90681d784a399c in 1304ms, sequenceid=395, compaction requested=true 2024-12-09T08:15:34,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:34,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:34,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-09T08:15:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-09T08:15:34,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-09T08:15:34,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3880 sec 2024-12-09T08:15:34,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.4020 sec 2024-12-09T08:15:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:34,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T08:15:34,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732133545, max timestamp=1733732134703Num uniques 859; 2024-12-09T08:15:34,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732133545, max timestamp=1733732134703Num uniques 859; 2024-12-09T08:15:34,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732133545, max timestamp=1733732134703Num uniques 859; 2024-12-09T08:15:34,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:34,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:34,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:34,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:34,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:34,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:34,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:34,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:34,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:34,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/f08d0a7a3a4c43eb896c59ce9d5a599d is 50, key is test_row_0/A:col10/1733732134703/Put/seqid=0 2024-12-09T08:15:34,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742144_1320 (size=12301) 2024-12-09T08:15:34,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732134740, max timestamp=1733732134747Num uniques 859; 2024-12-09T08:15:34,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732134740, max timestamp=1733732134747Num uniques 859; 2024-12-09T08:15:34,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732134740, max timestamp=1733732134747Num uniques 859; 2024-12-09T08:15:34,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:34,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:34,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:34,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733732134773, max timestamp=1733732134785Num uniques 1018; 2024-12-09T08:15:34,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T08:15:34,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733732134773, max timestamp=1733732134785Num uniques 1018; 2024-12-09T08:15:34,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T08:15:34,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733732134773, max timestamp=1733732134785Num uniques 1018; 2024-12-09T08:15:34,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T08:15:34,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733732134802, max timestamp=1733732134828Num uniques 1027; 2024-12-09T08:15:34,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T08:15:34,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733732134802, max timestamp=1733732134828Num uniques 1027; 2024-12-09T08:15:34,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T08:15:34,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733732134802, max timestamp=1733732134828Num uniques 1027; 2024-12-09T08:15:34,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T08:15:34,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733732134863, max timestamp=1733732134877Num uniques 1086; 2024-12-09T08:15:34,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-09T08:15:34,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733732134863, max timestamp=1733732134877Num uniques 1086; 2024-12-09T08:15:34,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-09T08:15:34,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733732134863, max timestamp=1733732134877Num uniques 1086; 2024-12-09T08:15:34,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-09T08:15:34,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733732134904, max timestamp=1733732134926Num uniques 1045; 2024-12-09T08:15:34,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-09T08:15:34,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733732134904, max timestamp=1733732134926Num uniques 1045; 2024-12-09T08:15:34,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-09T08:15:34,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733732134904, max timestamp=1733732134926Num uniques 1045; 2024-12-09T08:15:34,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-09T08:15:34,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733732134952, max timestamp=1733732134977Num uniques 1104; 2024-12-09T08:15:34,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-09T08:15:34,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733732134952, max timestamp=1733732134977Num uniques 1104; 2024-12-09T08:15:34,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733732134952, max timestamp=1733732134977Num uniques 1104; 2024-12-09T08:15:34,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-09T08:15:34,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-09T08:15:35,006 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-12-09T08:15:35,006 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:35,006 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,006 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:35,007 INFO [Thread-1135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1135. 2024-12-09T08:15:35,007 DEBUG [Thread-1135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:35,007 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,007 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:35,007 INFO [Thread-1139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1139. 2024-12-09T08:15:35,007 DEBUG [Thread-1139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:35,007 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:35,008 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-12-09T08:15:35,008 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,008 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:35,008 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,008 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:35,011 INFO [Thread-1141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1141. 2024-12-09T08:15:35,011 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-12-09T08:15:35,011 DEBUG [Thread-1141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:35,011 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:35,011 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,011 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,011 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:35,011 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:35,013 INFO [Thread-1133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1133. 2024-12-09T08:15:35,013 DEBUG [Thread-1133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:35,014 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,014 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:35,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:35,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/f08d0a7a3a4c43eb896c59ce9d5a599d 2024-12-09T08:15:35,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/27b2c615556a4f6994610126e40a6698 is 50, key is test_row_0/B:col10/1733732134703/Put/seqid=0 2024-12-09T08:15:35,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742145_1321 (size=12301) 2024-12-09T08:15:35,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/27b2c615556a4f6994610126e40a6698 2024-12-09T08:15:35,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/ca4f01f90ede4cb2941c74d525f7a218 is 50, key is test_row_0/C:col10/1733732134703/Put/seqid=0 2024-12-09T08:15:35,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742146_1322 (size=12301) 2024-12-09T08:15:35,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/ca4f01f90ede4cb2941c74d525f7a218 2024-12-09T08:15:35,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/f08d0a7a3a4c43eb896c59ce9d5a599d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/f08d0a7a3a4c43eb896c59ce9d5a599d 2024-12-09T08:15:35,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/f08d0a7a3a4c43eb896c59ce9d5a599d, entries=150, sequenceid=417, filesize=12.0 K 2024-12-09T08:15:35,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/27b2c615556a4f6994610126e40a6698 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/27b2c615556a4f6994610126e40a6698 2024-12-09T08:15:35,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/27b2c615556a4f6994610126e40a6698, entries=150, sequenceid=417, filesize=12.0 K 2024-12-09T08:15:35,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/ca4f01f90ede4cb2941c74d525f7a218 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/ca4f01f90ede4cb2941c74d525f7a218 2024-12-09T08:15:35,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/ca4f01f90ede4cb2941c74d525f7a218, entries=150, sequenceid=417, filesize=12.0 K 2024-12-09T08:15:35,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=114.05 KB/116790 for d33b04878a7a85043c90681d784a399c in 871ms, sequenceid=417, compaction requested=true 2024-12-09T08:15:35,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:35,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:35,576 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:15:35,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:35,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:35,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:35,576 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:15:35,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d33b04878a7a85043c90681d784a399c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:35,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:35,580 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:15:35,580 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/B is initiating minor compaction (all files) 2024-12-09T08:15:35,580 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/B in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:35,580 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/09fe20b69013423ca43c0eb55e7f0450, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a6aab1cb1a89499385262690f52ed9a3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/51908211018c4ec2aad3d183667d303c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/27b2c615556a4f6994610126e40a6698] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=48.8 K 2024-12-09T08:15:35,580 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52428 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:15:35,580 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/A is initiating minor compaction (all files) 2024-12-09T08:15:35,580 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/A in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:35,581 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/7b6e1227e364465f91cc779cb1059890, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/1efa76881a1f4c9ca3051a3ff3cd274a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/79cfc4d6e9a74c8e8f1044a655adcef2, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/f08d0a7a3a4c43eb896c59ce9d5a599d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=51.2 K 2024-12-09T08:15:35,581 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 09fe20b69013423ca43c0eb55e7f0450, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733732132214 2024-12-09T08:15:35,582 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b6e1227e364465f91cc779cb1059890, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733732132214 2024-12-09T08:15:35,583 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a6aab1cb1a89499385262690f52ed9a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733732132319 2024-12-09T08:15:35,583 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1efa76881a1f4c9ca3051a3ff3cd274a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733732132319 2024-12-09T08:15:35,583 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79cfc4d6e9a74c8e8f1044a655adcef2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733732132352 2024-12-09T08:15:35,583 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 51908211018c4ec2aad3d183667d303c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733732132352 2024-12-09T08:15:35,584 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting f08d0a7a3a4c43eb896c59ce9d5a599d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733732133545 2024-12-09T08:15:35,585 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 27b2c615556a4f6994610126e40a6698, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733732133545 2024-12-09T08:15:35,598 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#A#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:35,598 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c1eac1d03fad429e9ef1e8e55df1c9e8 is 50, key is test_row_0/A:col10/1733732134703/Put/seqid=0 2024-12-09T08:15:35,607 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#B#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:35,608 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/8603958148d444648715dc8b9b6d53da is 50, key is test_row_0/B:col10/1733732134703/Put/seqid=0 2024-12-09T08:15:35,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742147_1323 (size=13221) 2024-12-09T08:15:35,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742148_1324 (size=13221) 2024-12-09T08:15:35,633 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/c1eac1d03fad429e9ef1e8e55df1c9e8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c1eac1d03fad429e9ef1e8e55df1c9e8 2024-12-09T08:15:35,639 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d33b04878a7a85043c90681d784a399c/A of d33b04878a7a85043c90681d784a399c into c1eac1d03fad429e9ef1e8e55df1c9e8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:35,639 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:35,639 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/A, priority=12, startTime=1733732135575; duration=0sec 2024-12-09T08:15:35,640 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:35,640 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:A 2024-12-09T08:15:35,640 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:15:35,641 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:15:35,641 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): d33b04878a7a85043c90681d784a399c/C is initiating minor compaction (all files) 2024-12-09T08:15:35,641 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d33b04878a7a85043c90681d784a399c/C in TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:35,641 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/ffe3f71941b74518beb91578348a9abe, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/cf211007703e49559e219b5325b082d5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/e654ae0f183d4a80810b5772e2e68fd2, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/ca4f01f90ede4cb2941c74d525f7a218] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp, totalSize=48.8 K 2024-12-09T08:15:35,642 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffe3f71941b74518beb91578348a9abe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733732132214 2024-12-09T08:15:35,642 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf211007703e49559e219b5325b082d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733732132319 2024-12-09T08:15:35,643 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e654ae0f183d4a80810b5772e2e68fd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733732132352 2024-12-09T08:15:35,643 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca4f01f90ede4cb2941c74d525f7a218, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733732133545 2024-12-09T08:15:35,651 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d33b04878a7a85043c90681d784a399c#C#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:35,651 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/75681783e6c3447882fa7a8f1a02c36d is 50, key is test_row_0/C:col10/1733732134703/Put/seqid=0 2024-12-09T08:15:35,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742149_1325 (size=13221) 2024-12-09T08:15:36,030 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/8603958148d444648715dc8b9b6d53da as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/8603958148d444648715dc8b9b6d53da 2024-12-09T08:15:36,034 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d33b04878a7a85043c90681d784a399c/B of d33b04878a7a85043c90681d784a399c into 8603958148d444648715dc8b9b6d53da(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:36,034 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:36,034 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/B, priority=12, startTime=1733732135576; duration=0sec 2024-12-09T08:15:36,034 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:36,034 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:B 2024-12-09T08:15:36,061 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/75681783e6c3447882fa7a8f1a02c36d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/75681783e6c3447882fa7a8f1a02c36d 2024-12-09T08:15:36,068 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d33b04878a7a85043c90681d784a399c/C of d33b04878a7a85043c90681d784a399c into 75681783e6c3447882fa7a8f1a02c36d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:36,068 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:36,068 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., storeName=d33b04878a7a85043c90681d784a399c/C, priority=12, startTime=1733732135576; duration=0sec 2024-12-09T08:15:36,068 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:36,068 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d33b04878a7a85043c90681d784a399c:C 2024-12-09T08:15:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T08:15:36,401 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:36,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=120.76 KB heapSize=340.92 KB 2024-12-09T08:15:36,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:36,942 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-12-09T08:15:36,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:36,942 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:36,942 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:36,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:36,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:36,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:36,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:36,943 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:36,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:36,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/a325ab1753314ca2af006f9dce595aaf is 50, key is test_row_0/A:col10/1733732135002/Put/seqid=0 2024-12-09T08:15:36,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742150_1326 (size=12301) 2024-12-09T08:15:36,953 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-12-09T08:15:36,953 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:36,953 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:36,953 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:36,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:37,015 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-12-09T08:15:37,015 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:15:37,015 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:37,016 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:37,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 191 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 202 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3981 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4004 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3957 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4008 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3976 2024-12-09T08:15:37,016 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T08:15:37,017 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T08:15:37,017 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:15:37,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:37,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:37,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:15:37,018 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T08:15:37,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T08:15:37,023 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732137023"}]},"ts":"1733732137023"} 2024-12-09T08:15:37,025 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T08:15:37,025 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T08:15:37,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T08:15:37,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d33b04878a7a85043c90681d784a399c, UNASSIGN}] 2024-12-09T08:15:37,030 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d33b04878a7a85043c90681d784a399c, UNASSIGN 2024-12-09T08:15:37,031 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=d33b04878a7a85043c90681d784a399c, regionState=CLOSING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:15:37,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d33b04878a7a85043c90681d784a399c, UNASSIGN because future has completed 2024-12-09T08:15:37,034 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:15:37,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:15:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T08:15:37,186 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:37,187 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:15:37,187 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing d33b04878a7a85043c90681d784a399c, disabling compactions & flushes 2024-12-09T08:15:37,187 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T08:15:37,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/a325ab1753314ca2af006f9dce595aaf 2024-12-09T08:15:37,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/e47d260cefe44e5b98ae4f2632e091c6 is 50, key is test_row_0/B:col10/1733732135002/Put/seqid=0 2024-12-09T08:15:37,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742151_1327 (size=12301) 2024-12-09T08:15:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T08:15:37,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/e47d260cefe44e5b98ae4f2632e091c6 2024-12-09T08:15:37,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/a17f9535a525446c92a4e200b124208d is 50, key is test_row_0/C:col10/1733732135002/Put/seqid=0 2024-12-09T08:15:37,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742152_1328 (size=12301) 2024-12-09T08:15:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T08:15:38,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/a17f9535a525446c92a4e200b124208d 2024-12-09T08:15:38,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/a325ab1753314ca2af006f9dce595aaf as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/a325ab1753314ca2af006f9dce595aaf 2024-12-09T08:15:38,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/a325ab1753314ca2af006f9dce595aaf, entries=150, sequenceid=540, filesize=12.0 K 2024-12-09T08:15:38,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/e47d260cefe44e5b98ae4f2632e091c6 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e47d260cefe44e5b98ae4f2632e091c6 2024-12-09T08:15:38,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e47d260cefe44e5b98ae4f2632e091c6, entries=150, sequenceid=540, filesize=12.0 K 2024-12-09T08:15:38,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/a17f9535a525446c92a4e200b124208d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a17f9535a525446c92a4e200b124208d 2024-12-09T08:15:38,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a17f9535a525446c92a4e200b124208d, entries=150, sequenceid=540, filesize=12.0 K 2024-12-09T08:15:38,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.88 KB/349056, currentSize=13.42 KB/13740 for d33b04878a7a85043c90681d784a399c in 1266ms, sequenceid=540, compaction requested=false 2024-12-09T08:15:38,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d33b04878a7a85043c90681d784a399c: 2024-12-09T08:15:38,208 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:38,208 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:38,208 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. after waiting 0 ms 2024-12-09T08:15:38,208 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:38,208 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing d33b04878a7a85043c90681d784a399c 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-09T08:15:38,208 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=A 2024-12-09T08:15:38,209 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:38,209 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=B 2024-12-09T08:15:38,209 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:38,209 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d33b04878a7a85043c90681d784a399c, store=C 2024-12-09T08:15:38,209 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:38,212 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/5974cb70c99d42bebd40360d2a24d9b2 is 50, key is test_row_2/A:col10/1733732137012/Put/seqid=0 2024-12-09T08:15:38,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742153_1329 (size=7415) 2024-12-09T08:15:38,615 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/5974cb70c99d42bebd40360d2a24d9b2 2024-12-09T08:15:38,625 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/8d51353510bc4082bdced187d213cf59 is 50, key is test_row_2/B:col10/1733732137012/Put/seqid=0 2024-12-09T08:15:38,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742154_1330 (size=7415) 2024-12-09T08:15:39,030 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/8d51353510bc4082bdced187d213cf59 2024-12-09T08:15:39,038 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/c2ddb77b01d249e09528ea77af86c785 is 50, key is test_row_2/C:col10/1733732137012/Put/seqid=0 2024-12-09T08:15:39,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742155_1331 (size=7415) 2024-12-09T08:15:39,059 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/c2ddb77b01d249e09528ea77af86c785 2024-12-09T08:15:39,064 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/A/5974cb70c99d42bebd40360d2a24d9b2 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/5974cb70c99d42bebd40360d2a24d9b2 2024-12-09T08:15:39,069 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/5974cb70c99d42bebd40360d2a24d9b2, entries=50, sequenceid=545, filesize=7.2 K 2024-12-09T08:15:39,069 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/B/8d51353510bc4082bdced187d213cf59 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/8d51353510bc4082bdced187d213cf59 2024-12-09T08:15:39,073 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/8d51353510bc4082bdced187d213cf59, entries=50, sequenceid=545, filesize=7.2 K 2024-12-09T08:15:39,074 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/.tmp/C/c2ddb77b01d249e09528ea77af86c785 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/c2ddb77b01d249e09528ea77af86c785 2024-12-09T08:15:39,079 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/c2ddb77b01d249e09528ea77af86c785, entries=50, sequenceid=545, filesize=7.2 K 2024-12-09T08:15:39,079 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for d33b04878a7a85043c90681d784a399c in 871ms, sequenceid=545, compaction requested=true 2024-12-09T08:15:39,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/9bb5966dd3b347a1a0aa127a036331c5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/bd4760e9a1b448aba4e871359d7ce4b3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/1cede18eeb0e4218bbce903b8e26689b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/e8f9e797c0bc405db1ca0f0fbee59656, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/6c3275b6f7a84a6fa9422f90bf2685f4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/51f46e8c917d45469140113dc6261352, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/f481c9893ac74c9097a795d033611993, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/da488ff561d34591b61d4372a2712bd4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/5ee01ae01b8049d2be05eb8335adfb87, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c99ab3fe9f404e31a3c0c706fa1c5455, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/089a42e3aa1c44b8bec76eccd30c4148, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c6a7410789e94e1fa5c16cff248ba59c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c2d7b2e169ab4614870ed50e2573886b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/ed0bcfd0e6bd41df84c13751a7aad08e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/4d9038862bc945c0a43747fffcef7f39, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/9db5c105f16c42479f7bdadc57697d1b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/3f7d9ad6a82b4ce686e2000540feefda, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/98225ab92a2d4fadbac04531abc9d9fb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/e13ee9d7f2d542b0bf59ef9441af133b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/7c7f9d9bf99c43fcbab91529c48d1af3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c68e704638d04b35a88eecc1e8f5e4fa, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/edaaba0f32474257a729557fe10e3a50, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/7b6e1227e364465f91cc779cb1059890, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/ded50c40ab5a4b8aa98d84fdb52d344d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/1efa76881a1f4c9ca3051a3ff3cd274a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/79cfc4d6e9a74c8e8f1044a655adcef2, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/f08d0a7a3a4c43eb896c59ce9d5a599d] to archive 2024-12-09T08:15:39,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:15:39,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/9bb5966dd3b347a1a0aa127a036331c5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/9bb5966dd3b347a1a0aa127a036331c5 2024-12-09T08:15:39,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/bd4760e9a1b448aba4e871359d7ce4b3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/bd4760e9a1b448aba4e871359d7ce4b3 2024-12-09T08:15:39,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/1cede18eeb0e4218bbce903b8e26689b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/1cede18eeb0e4218bbce903b8e26689b 2024-12-09T08:15:39,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/e8f9e797c0bc405db1ca0f0fbee59656 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/e8f9e797c0bc405db1ca0f0fbee59656 2024-12-09T08:15:39,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/6c3275b6f7a84a6fa9422f90bf2685f4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/6c3275b6f7a84a6fa9422f90bf2685f4 2024-12-09T08:15:39,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/51f46e8c917d45469140113dc6261352 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/51f46e8c917d45469140113dc6261352 2024-12-09T08:15:39,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/f481c9893ac74c9097a795d033611993 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/f481c9893ac74c9097a795d033611993 2024-12-09T08:15:39,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/da488ff561d34591b61d4372a2712bd4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/da488ff561d34591b61d4372a2712bd4 2024-12-09T08:15:39,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/5ee01ae01b8049d2be05eb8335adfb87 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/5ee01ae01b8049d2be05eb8335adfb87 2024-12-09T08:15:39,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c99ab3fe9f404e31a3c0c706fa1c5455 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c99ab3fe9f404e31a3c0c706fa1c5455 2024-12-09T08:15:39,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/089a42e3aa1c44b8bec76eccd30c4148 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/089a42e3aa1c44b8bec76eccd30c4148 2024-12-09T08:15:39,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c6a7410789e94e1fa5c16cff248ba59c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c6a7410789e94e1fa5c16cff248ba59c 2024-12-09T08:15:39,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c2d7b2e169ab4614870ed50e2573886b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c2d7b2e169ab4614870ed50e2573886b 2024-12-09T08:15:39,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/ed0bcfd0e6bd41df84c13751a7aad08e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/ed0bcfd0e6bd41df84c13751a7aad08e 2024-12-09T08:15:39,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/4d9038862bc945c0a43747fffcef7f39 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/4d9038862bc945c0a43747fffcef7f39 2024-12-09T08:15:39,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/9db5c105f16c42479f7bdadc57697d1b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/9db5c105f16c42479f7bdadc57697d1b 2024-12-09T08:15:39,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/3f7d9ad6a82b4ce686e2000540feefda to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/3f7d9ad6a82b4ce686e2000540feefda 2024-12-09T08:15:39,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/98225ab92a2d4fadbac04531abc9d9fb to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/98225ab92a2d4fadbac04531abc9d9fb 2024-12-09T08:15:39,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/e13ee9d7f2d542b0bf59ef9441af133b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/e13ee9d7f2d542b0bf59ef9441af133b 2024-12-09T08:15:39,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/7c7f9d9bf99c43fcbab91529c48d1af3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/7c7f9d9bf99c43fcbab91529c48d1af3 2024-12-09T08:15:39,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c68e704638d04b35a88eecc1e8f5e4fa to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c68e704638d04b35a88eecc1e8f5e4fa 2024-12-09T08:15:39,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/edaaba0f32474257a729557fe10e3a50 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/edaaba0f32474257a729557fe10e3a50 2024-12-09T08:15:39,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/7b6e1227e364465f91cc779cb1059890 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/7b6e1227e364465f91cc779cb1059890 2024-12-09T08:15:39,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/ded50c40ab5a4b8aa98d84fdb52d344d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/ded50c40ab5a4b8aa98d84fdb52d344d 2024-12-09T08:15:39,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/1efa76881a1f4c9ca3051a3ff3cd274a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/1efa76881a1f4c9ca3051a3ff3cd274a 2024-12-09T08:15:39,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/79cfc4d6e9a74c8e8f1044a655adcef2 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/79cfc4d6e9a74c8e8f1044a655adcef2 2024-12-09T08:15:39,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/f08d0a7a3a4c43eb896c59ce9d5a599d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/f08d0a7a3a4c43eb896c59ce9d5a599d 2024-12-09T08:15:39,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/140b40dc6ae641cd80c7408f92c023c2, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/61b5fa7186cc414aa73615dfe7b5d77b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e3cfd7dfb3464bc49e99417bcab24f68, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/0e41c4a2a8ac445f8ece62b67160388f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/f11fcc1cd4074b30afe2b3e7920efa51, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a6336a6337ba48b2804a2d34ed75fefe, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/2a11dceb43d54681a51de185fee9953b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/0e78547564544574ac4654d11b639070, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/d178cce8fc2d41d49035689e5122ffe8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/1c55e7f9e2d845f5a944835e36f08435, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/03ed33720a3f4269b1498fa63250ba71, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/9a1f4d334e084fe4b4f70d8e3461b204, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e21ad95fa1cc4a6e88c533faf868814d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/ba1e5736467544beb9c131735f086164, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a2f15a2db9d545d1af809e12c7cccb05, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/eba030051ef645f3bba69b62bbf7fd13, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/48a281d7b6a446949388b708c4f211e0, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/d83cd970fca9492db1c202fe7c1865ec, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a9e1e6bca3274b3fb558cb3ac8cd379e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/26d45d8dfbbd42ba8ec7c33929bf53be, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/10c6fafa6ef1468a8b6ad6df6ca0ed0f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a3b31def2f784f9cb2716175fbf24c60, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/09fe20b69013423ca43c0eb55e7f0450, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/19286c57bd1d4494ba3f16f6238cb10b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a6aab1cb1a89499385262690f52ed9a3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/51908211018c4ec2aad3d183667d303c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/27b2c615556a4f6994610126e40a6698] to archive 2024-12-09T08:15:39,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:15:39,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/140b40dc6ae641cd80c7408f92c023c2 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/140b40dc6ae641cd80c7408f92c023c2 2024-12-09T08:15:39,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/61b5fa7186cc414aa73615dfe7b5d77b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/61b5fa7186cc414aa73615dfe7b5d77b 2024-12-09T08:15:39,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e3cfd7dfb3464bc49e99417bcab24f68 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e3cfd7dfb3464bc49e99417bcab24f68 2024-12-09T08:15:39,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/0e41c4a2a8ac445f8ece62b67160388f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/0e41c4a2a8ac445f8ece62b67160388f 2024-12-09T08:15:39,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/f11fcc1cd4074b30afe2b3e7920efa51 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/f11fcc1cd4074b30afe2b3e7920efa51 2024-12-09T08:15:39,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a6336a6337ba48b2804a2d34ed75fefe to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a6336a6337ba48b2804a2d34ed75fefe 2024-12-09T08:15:39,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/2a11dceb43d54681a51de185fee9953b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/2a11dceb43d54681a51de185fee9953b 2024-12-09T08:15:39,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/0e78547564544574ac4654d11b639070 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/0e78547564544574ac4654d11b639070 2024-12-09T08:15:39,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/d178cce8fc2d41d49035689e5122ffe8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/d178cce8fc2d41d49035689e5122ffe8 2024-12-09T08:15:39,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/1c55e7f9e2d845f5a944835e36f08435 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/1c55e7f9e2d845f5a944835e36f08435 2024-12-09T08:15:39,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/03ed33720a3f4269b1498fa63250ba71 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/03ed33720a3f4269b1498fa63250ba71 2024-12-09T08:15:39,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/9a1f4d334e084fe4b4f70d8e3461b204 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/9a1f4d334e084fe4b4f70d8e3461b204 2024-12-09T08:15:39,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e21ad95fa1cc4a6e88c533faf868814d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e21ad95fa1cc4a6e88c533faf868814d 2024-12-09T08:15:39,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/ba1e5736467544beb9c131735f086164 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/ba1e5736467544beb9c131735f086164 2024-12-09T08:15:39,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a2f15a2db9d545d1af809e12c7cccb05 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a2f15a2db9d545d1af809e12c7cccb05 2024-12-09T08:15:39,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/eba030051ef645f3bba69b62bbf7fd13 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/eba030051ef645f3bba69b62bbf7fd13 2024-12-09T08:15:39,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/48a281d7b6a446949388b708c4f211e0 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/48a281d7b6a446949388b708c4f211e0 2024-12-09T08:15:39,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/d83cd970fca9492db1c202fe7c1865ec to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/d83cd970fca9492db1c202fe7c1865ec 2024-12-09T08:15:39,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a9e1e6bca3274b3fb558cb3ac8cd379e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a9e1e6bca3274b3fb558cb3ac8cd379e 2024-12-09T08:15:39,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/26d45d8dfbbd42ba8ec7c33929bf53be to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/26d45d8dfbbd42ba8ec7c33929bf53be 2024-12-09T08:15:39,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/10c6fafa6ef1468a8b6ad6df6ca0ed0f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/10c6fafa6ef1468a8b6ad6df6ca0ed0f 2024-12-09T08:15:39,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a3b31def2f784f9cb2716175fbf24c60 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a3b31def2f784f9cb2716175fbf24c60 2024-12-09T08:15:39,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/09fe20b69013423ca43c0eb55e7f0450 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/09fe20b69013423ca43c0eb55e7f0450 2024-12-09T08:15:39,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/19286c57bd1d4494ba3f16f6238cb10b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/19286c57bd1d4494ba3f16f6238cb10b 2024-12-09T08:15:39,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a6aab1cb1a89499385262690f52ed9a3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/a6aab1cb1a89499385262690f52ed9a3 2024-12-09T08:15:39,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/51908211018c4ec2aad3d183667d303c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/51908211018c4ec2aad3d183667d303c 2024-12-09T08:15:39,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/27b2c615556a4f6994610126e40a6698 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/27b2c615556a4f6994610126e40a6698 2024-12-09T08:15:39,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/0f183aaac9f942218266d1c2606c95df, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/9c1e645359a74031b96c7a65e01cf5e5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/4e44b24fb35e4ed98c36450446e6098e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/d590819bb2b343afb38c480c88e40d8d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/165bb059dd994912859783a1f41caf74, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/766ca32c95284009b7802293ccbc0bbd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5af98effa16a43ce87548de796337ebe, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/e9055227fca54129a1d3ecd96c73b2e9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/2ba257ea779e4a2c8fcf84c980e1a782, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/f5edbffd8ebd48e08eb34b46039400b4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/d1b8f228c9f34987a703e41fb47f68b3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/1d4b4c54792641569d0a0034b800a134, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a6913be9d4eb4399a355ad29c1274166, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/8c604ae8cb5546d8b47dbfff48b798e4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/702d9200e027424a902fbe2eff4f87b6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5beae54798704d3596ca62d1c70bccb0, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5dd24cf5cd634e71bdcced03f43bc6e9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5d574b21a3c24143ab09b77db56dd53f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/944bd484228d41f29647506ade1abbdc, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/04e76b3fa44d4dcfb4cdcc9250de9b18, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/6e20afa89a924085b4336539e46b2f3c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a16a9b8f4cc04eb58aa9362c57e8fd15, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/ffe3f71941b74518beb91578348a9abe, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/0642335f68664043a244c0219ad87bd4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/cf211007703e49559e219b5325b082d5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/e654ae0f183d4a80810b5772e2e68fd2, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/ca4f01f90ede4cb2941c74d525f7a218] to archive 2024-12-09T08:15:39,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:15:39,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/0f183aaac9f942218266d1c2606c95df to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/0f183aaac9f942218266d1c2606c95df 2024-12-09T08:15:39,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/9c1e645359a74031b96c7a65e01cf5e5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/9c1e645359a74031b96c7a65e01cf5e5 2024-12-09T08:15:39,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/4e44b24fb35e4ed98c36450446e6098e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/4e44b24fb35e4ed98c36450446e6098e 2024-12-09T08:15:39,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/d590819bb2b343afb38c480c88e40d8d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/d590819bb2b343afb38c480c88e40d8d 2024-12-09T08:15:39,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/165bb059dd994912859783a1f41caf74 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/165bb059dd994912859783a1f41caf74 2024-12-09T08:15:39,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/766ca32c95284009b7802293ccbc0bbd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/766ca32c95284009b7802293ccbc0bbd 2024-12-09T08:15:39,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5af98effa16a43ce87548de796337ebe to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5af98effa16a43ce87548de796337ebe 2024-12-09T08:15:39,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/e9055227fca54129a1d3ecd96c73b2e9 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/e9055227fca54129a1d3ecd96c73b2e9 2024-12-09T08:15:39,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/2ba257ea779e4a2c8fcf84c980e1a782 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/2ba257ea779e4a2c8fcf84c980e1a782 2024-12-09T08:15:39,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/f5edbffd8ebd48e08eb34b46039400b4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/f5edbffd8ebd48e08eb34b46039400b4 2024-12-09T08:15:39,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/d1b8f228c9f34987a703e41fb47f68b3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/d1b8f228c9f34987a703e41fb47f68b3 2024-12-09T08:15:39,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/1d4b4c54792641569d0a0034b800a134 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/1d4b4c54792641569d0a0034b800a134 2024-12-09T08:15:39,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a6913be9d4eb4399a355ad29c1274166 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a6913be9d4eb4399a355ad29c1274166 2024-12-09T08:15:39,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/8c604ae8cb5546d8b47dbfff48b798e4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/8c604ae8cb5546d8b47dbfff48b798e4 2024-12-09T08:15:39,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/702d9200e027424a902fbe2eff4f87b6 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/702d9200e027424a902fbe2eff4f87b6 2024-12-09T08:15:39,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5beae54798704d3596ca62d1c70bccb0 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5beae54798704d3596ca62d1c70bccb0 2024-12-09T08:15:39,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5dd24cf5cd634e71bdcced03f43bc6e9 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5dd24cf5cd634e71bdcced03f43bc6e9 2024-12-09T08:15:39,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5d574b21a3c24143ab09b77db56dd53f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/5d574b21a3c24143ab09b77db56dd53f 2024-12-09T08:15:39,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/944bd484228d41f29647506ade1abbdc to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/944bd484228d41f29647506ade1abbdc 2024-12-09T08:15:39,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/04e76b3fa44d4dcfb4cdcc9250de9b18 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/04e76b3fa44d4dcfb4cdcc9250de9b18 2024-12-09T08:15:39,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/6e20afa89a924085b4336539e46b2f3c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/6e20afa89a924085b4336539e46b2f3c 2024-12-09T08:15:39,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a16a9b8f4cc04eb58aa9362c57e8fd15 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a16a9b8f4cc04eb58aa9362c57e8fd15 2024-12-09T08:15:39,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/ffe3f71941b74518beb91578348a9abe to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/ffe3f71941b74518beb91578348a9abe 2024-12-09T08:15:39,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/0642335f68664043a244c0219ad87bd4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/0642335f68664043a244c0219ad87bd4 2024-12-09T08:15:39,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/cf211007703e49559e219b5325b082d5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/cf211007703e49559e219b5325b082d5 2024-12-09T08:15:39,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/e654ae0f183d4a80810b5772e2e68fd2 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/e654ae0f183d4a80810b5772e2e68fd2 2024-12-09T08:15:39,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/ca4f01f90ede4cb2941c74d525f7a218 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/ca4f01f90ede4cb2941c74d525f7a218 2024-12-09T08:15:39,170 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/recovered.edits/548.seqid, newMaxSeqId=548, maxSeqId=1 2024-12-09T08:15:39,170 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. 2024-12-09T08:15:39,170 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for d33b04878a7a85043c90681d784a399c: Waiting for close lock at 1733732137187Running coprocessor pre-close hooks at 1733732137187Disabling compacts and flushes for region at 1733732137187Disabling writes for close at 1733732138208 (+1021 ms)Obtaining lock to block concurrent updates at 1733732138208Preparing flush snapshotting stores in d33b04878a7a85043c90681d784a399c at 1733732138208Finished memstore snapshotting TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733732138209 (+1 ms)Flushing stores of TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c. at 1733732138209Flushing d33b04878a7a85043c90681d784a399c/A: creating writer at 1733732138209Flushing d33b04878a7a85043c90681d784a399c/A: appending metadata at 1733732138211 (+2 ms)Flushing d33b04878a7a85043c90681d784a399c/A: closing flushed file at 1733732138211Flushing d33b04878a7a85043c90681d784a399c/B: creating writer at 1733732138620 (+409 ms)Flushing d33b04878a7a85043c90681d784a399c/B: appending metadata at 1733732138624 (+4 ms)Flushing d33b04878a7a85043c90681d784a399c/B: closing flushed file at 1733732138624Flushing d33b04878a7a85043c90681d784a399c/C: creating writer at 1733732139034 (+410 ms)Flushing d33b04878a7a85043c90681d784a399c/C: appending metadata at 1733732139037 (+3 ms)Flushing d33b04878a7a85043c90681d784a399c/C: closing flushed file at 1733732139037Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f3b811e: reopening flushed file at 1733732139064 (+27 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ce7d806: reopening flushed file at 1733732139069 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f0cf625: reopening flushed file at 1733732139073 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for d33b04878a7a85043c90681d784a399c in 871ms, sequenceid=545, compaction requested=true at 1733732139080 (+7 ms)Writing region close event to WAL at 1733732139167 (+87 ms)Running coprocessor post-close hooks at 1733732139170 (+3 ms)Closed at 1733732139170 2024-12-09T08:15:39,172 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:39,172 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=d33b04878a7a85043c90681d784a399c, regionState=CLOSED 2024-12-09T08:15:39,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:15:39,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-09T08:15:39,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure d33b04878a7a85043c90681d784a399c, server=5bda946059d1,43255,1733732044053 in 2.1400 sec 2024-12-09T08:15:39,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-09T08:15:39,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d33b04878a7a85043c90681d784a399c, UNASSIGN in 2.1460 sec 2024-12-09T08:15:39,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-09T08:15:39,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1510 sec 2024-12-09T08:15:39,179 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732139179"}]},"ts":"1733732139179"} 2024-12-09T08:15:39,181 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T08:15:39,181 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T08:15:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T08:15:39,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1640 sec 2024-12-09T08:15:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T08:15:41,202 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:41,203 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T08:15:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:41,207 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T08:15:41,209 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:41,213 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:41,217 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/recovered.edits] 2024-12-09T08:15:41,221 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/5974cb70c99d42bebd40360d2a24d9b2 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/5974cb70c99d42bebd40360d2a24d9b2 2024-12-09T08:15:41,222 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/a325ab1753314ca2af006f9dce595aaf to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/a325ab1753314ca2af006f9dce595aaf 2024-12-09T08:15:41,223 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c1eac1d03fad429e9ef1e8e55df1c9e8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/A/c1eac1d03fad429e9ef1e8e55df1c9e8 2024-12-09T08:15:41,225 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/8603958148d444648715dc8b9b6d53da to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/8603958148d444648715dc8b9b6d53da 2024-12-09T08:15:41,226 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/8d51353510bc4082bdced187d213cf59 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/8d51353510bc4082bdced187d213cf59 2024-12-09T08:15:41,227 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e47d260cefe44e5b98ae4f2632e091c6 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/B/e47d260cefe44e5b98ae4f2632e091c6 2024-12-09T08:15:41,229 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/75681783e6c3447882fa7a8f1a02c36d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/75681783e6c3447882fa7a8f1a02c36d 2024-12-09T08:15:41,230 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a17f9535a525446c92a4e200b124208d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/a17f9535a525446c92a4e200b124208d 2024-12-09T08:15:41,231 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/c2ddb77b01d249e09528ea77af86c785 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/C/c2ddb77b01d249e09528ea77af86c785 2024-12-09T08:15:41,232 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/recovered.edits/548.seqid to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c/recovered.edits/548.seqid 2024-12-09T08:15:41,233 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/d33b04878a7a85043c90681d784a399c 2024-12-09T08:15:41,233 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T08:15:41,234 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:41,236 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T08:15:41,238 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T08:15:41,239 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:41,239 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T08:15:41,239 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733732141239"}]},"ts":"9223372036854775807"} 2024-12-09T08:15:41,240 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T08:15:41,240 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d33b04878a7a85043c90681d784a399c, NAME => 'TestAcidGuarantees,,1733732112814.d33b04878a7a85043c90681d784a399c.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T08:15:41,240 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T08:15:41,241 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733732141240"}]},"ts":"9223372036854775807"} 2024-12-09T08:15:41,242 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T08:15:41,242 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:41,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-12-09T08:15:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T08:15:41,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T08:15:41,312 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:41,322 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_646891914_22 at /127.0.0.1:58994 [Waiting for operation #345] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_189377618_22 at /127.0.0.1:60802 [Waiting for operation #222] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=397 (was 395) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4118 (was 4797) 2024-12-09T08:15:41,333 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=397, ProcessCount=11, AvailableMemoryMB=4117 2024-12-09T08:15:41,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T08:15:41,335 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:15:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:41,337 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T08:15:41,337 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:41,337 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-12-09T08:15:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T08:15:41,338 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T08:15:41,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742156_1332 (size=1061) 2024-12-09T08:15:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T08:15:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T08:15:41,745 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447 2024-12-09T08:15:41,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742157_1333 (size=53) 2024-12-09T08:15:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T08:15:42,151 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:15:42,151 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5081394aff7face3f62aba376aebd75e, disabling compactions & flushes 2024-12-09T08:15:42,151 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:42,151 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:42,151 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. after waiting 0 ms 2024-12-09T08:15:42,151 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:42,151 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:42,151 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5081394aff7face3f62aba376aebd75e: Waiting for close lock at 1733732142151Disabling compacts and flushes for region at 1733732142151Disabling writes for close at 1733732142151Writing region close event to WAL at 1733732142151Closed at 1733732142151 2024-12-09T08:15:42,153 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T08:15:42,153 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733732142153"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733732142153"}]},"ts":"1733732142153"} 2024-12-09T08:15:42,155 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T08:15:42,156 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T08:15:42,156 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732142156"}]},"ts":"1733732142156"} 2024-12-09T08:15:42,158 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T08:15:42,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, ASSIGN}] 2024-12-09T08:15:42,159 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, ASSIGN 2024-12-09T08:15:42,160 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, ASSIGN; state=OFFLINE, location=5bda946059d1,43255,1733732044053; forceNewPlan=false, retain=false 2024-12-09T08:15:42,310 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5081394aff7face3f62aba376aebd75e, regionState=OPENING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:15:42,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, ASSIGN because future has completed 2024-12-09T08:15:42,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:15:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T08:15:42,472 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:42,472 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:15:42,472 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,472 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:15:42,473 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,473 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,474 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,475 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:15:42,475 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5081394aff7face3f62aba376aebd75e columnFamilyName A 2024-12-09T08:15:42,475 DEBUG [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:42,476 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.HStore(327): Store=5081394aff7face3f62aba376aebd75e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:15:42,476 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,477 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:15:42,477 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5081394aff7face3f62aba376aebd75e columnFamilyName B 2024-12-09T08:15:42,477 DEBUG [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:42,478 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.HStore(327): Store=5081394aff7face3f62aba376aebd75e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:15:42,478 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,479 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:15:42,479 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5081394aff7face3f62aba376aebd75e columnFamilyName C 2024-12-09T08:15:42,479 DEBUG [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:42,479 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.HStore(327): Store=5081394aff7face3f62aba376aebd75e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:15:42,479 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:42,480 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,480 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,481 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,482 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,482 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,482 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:15:42,483 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,486 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:15:42,486 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 5081394aff7face3f62aba376aebd75e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67630421, jitterRate=0.007771804928779602}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:15:42,486 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:42,487 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 5081394aff7face3f62aba376aebd75e: Running coprocessor pre-open hook at 1733732142473Writing region info on filesystem at 1733732142473Initializing all the Stores at 1733732142473Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732142473Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732142474 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732142474Cleaning up temporary data from old regions at 1733732142482 (+8 ms)Running coprocessor post-open hooks at 1733732142486 (+4 ms)Region opened successfully at 1733732142487 (+1 ms) 2024-12-09T08:15:42,488 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., pid=85, masterSystemTime=1733732142464 2024-12-09T08:15:42,490 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:42,490 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:42,491 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5081394aff7face3f62aba376aebd75e, regionState=OPEN, openSeqNum=2, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:15:42,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:15:42,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-09T08:15:42,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 in 181 msec 2024-12-09T08:15:42,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-09T08:15:42,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, ASSIGN in 337 msec 2024-12-09T08:15:42,498 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T08:15:42,499 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732142498"}]},"ts":"1733732142498"} 2024-12-09T08:15:42,500 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T08:15:42,501 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T08:15:42,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-12-09T08:15:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T08:15:43,482 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:43,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:43,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:43,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:43,484 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43fbf0e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:43,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:43,485 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51365d56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:43,486 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:43,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:43,489 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T08:15:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 5bda946059d1,45611,1733732043366 2024-12-09T08:15:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1d65db38 2024-12-09T08:15:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T08:15:43,490 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T08:15:43,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T08:15:43,491 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:15:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T08:15:43,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742158_1334 (size=1097) 2024-12-09T08:15:43,906 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T08:15:43,906 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T08:15:43,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T08:15:43,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, REOPEN/MOVE}] 2024-12-09T08:15:43,913 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, REOPEN/MOVE 2024-12-09T08:15:43,916 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5081394aff7face3f62aba376aebd75e, regionState=CLOSING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:15:43,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, REOPEN/MOVE because future has completed 2024-12-09T08:15:43,918 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:15:43,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:15:44,071 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,071 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:15:44,071 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 5081394aff7face3f62aba376aebd75e, disabling compactions & flushes 2024-12-09T08:15:44,071 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,071 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,071 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. after waiting 0 ms 2024-12-09T08:15:44,071 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,075 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T08:15:44,075 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,075 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 5081394aff7face3f62aba376aebd75e: Waiting for close lock at 1733732144071Running coprocessor pre-close hooks at 1733732144071Disabling compacts and flushes for region at 1733732144071Disabling writes for close at 1733732144071Writing region close event to WAL at 1733732144071Running coprocessor post-close hooks at 1733732144075 (+4 ms)Closed at 1733732144075 2024-12-09T08:15:44,075 WARN [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 5081394aff7face3f62aba376aebd75e to self. 2024-12-09T08:15:44,077 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,077 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5081394aff7face3f62aba376aebd75e, regionState=CLOSED 2024-12-09T08:15:44,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:15:44,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-09T08:15:44,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 in 161 msec 2024-12-09T08:15:44,081 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, REOPEN/MOVE; state=CLOSED, location=5bda946059d1,43255,1733732044053; forceNewPlan=false, retain=true 2024-12-09T08:15:44,231 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5081394aff7face3f62aba376aebd75e, regionState=OPENING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, REOPEN/MOVE because future has completed 2024-12-09T08:15:44,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:15:44,388 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,388 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:15:44,388 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,388 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:15:44,388 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,388 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,390 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,391 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:15:44,391 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5081394aff7face3f62aba376aebd75e columnFamilyName A 2024-12-09T08:15:44,392 DEBUG [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:44,393 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.HStore(327): Store=5081394aff7face3f62aba376aebd75e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:15:44,393 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,394 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:15:44,394 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5081394aff7face3f62aba376aebd75e columnFamilyName B 2024-12-09T08:15:44,394 DEBUG [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:44,395 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.HStore(327): Store=5081394aff7face3f62aba376aebd75e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:15:44,395 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,395 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:15:44,395 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5081394aff7face3f62aba376aebd75e columnFamilyName C 2024-12-09T08:15:44,396 DEBUG [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:44,396 INFO [StoreOpener-5081394aff7face3f62aba376aebd75e-1 {}] regionserver.HStore(327): Store=5081394aff7face3f62aba376aebd75e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:15:44,396 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,396 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,397 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,398 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,399 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,400 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,400 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:15:44,402 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,403 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 5081394aff7face3f62aba376aebd75e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68984288, jitterRate=0.027945995330810547}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:15:44,403 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,404 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 5081394aff7face3f62aba376aebd75e: Running coprocessor pre-open hook at 1733732144388Writing region info on filesystem at 1733732144388Initializing all the Stores at 1733732144389 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732144389Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732144389Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732144389Cleaning up temporary data from old regions at 1733732144400 (+11 ms)Running coprocessor post-open hooks at 1733732144403 (+3 ms)Region opened successfully at 1733732144404 (+1 ms) 2024-12-09T08:15:44,405 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., pid=90, masterSystemTime=1733732144385 2024-12-09T08:15:44,407 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,407 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,407 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5081394aff7face3f62aba376aebd75e, regionState=OPEN, openSeqNum=5, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:15:44,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-12-09T08:15:44,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 in 177 msec 2024-12-09T08:15:44,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-09T08:15:44,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, REOPEN/MOVE in 502 msec 2024-12-09T08:15:44,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-09T08:15:44,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-12-09T08:15:44,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 928 msec 2024-12-09T08:15:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-09T08:15:44,423 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:44,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:44,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:44,425 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f247131, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:44,427 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20284d24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:44,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:44,429 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49460f40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:44,431 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ab69c84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:44,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:44,433 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@506b2976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:44,435 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f91b71f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:44,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:44,438 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b67b32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:44,439 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7634180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:44,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:44,441 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf9f10c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:44,443 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:44,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16db3480, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:44,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:44,445 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55715975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:44,446 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2309b442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:44,447 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:44,448 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7033eee3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:44,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:44,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:44,449 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d0b2e67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:44,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:44,451 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@783a221e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:44,452 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d551fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:44,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:44,453 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2473bbbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:44,454 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:44,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee7f16f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:15:44,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:15:44,456 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:15:44,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:15:44,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:15:44,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57394f8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:15:44,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:15:44,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:15:44,458 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:15:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b1e660, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:15:44,459 DEBUG [Thread-1536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:44,459 DEBUG [Thread-1534 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:44,459 DEBUG [Thread-1531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:44,459 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:44,459 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:44,460 DEBUG [Thread-1523 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:44,460 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:44,461 DEBUG [Thread-1529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:44,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:44,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:44,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:44,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:44,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:44,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:44,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:44,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:44,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-09T08:15:44,463 DEBUG [Thread-1525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:15:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T08:15:44,464 DEBUG [Thread-1527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:15:44,465 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:15:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:15:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:15:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:15:44,466 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:44,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:44,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:44,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:15:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:15:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:15:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:15:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:15:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:15:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:15:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:15:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:15:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:44,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120929785db219e540799e99c36fcfb358ea_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732144470/Put/seqid=0 2024-12-09T08:15:44,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742159_1335 (size=17034) 2024-12-09T08:15:44,506 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:44,510 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120929785db219e540799e99c36fcfb358ea_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120929785db219e540799e99c36fcfb358ea_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:44,511 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/25b59846fa2540189980aebe42e6719e, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:44,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/25b59846fa2540189980aebe42e6719e is 175, key is test_row_0/A:col10/1733732144470/Put/seqid=0 2024-12-09T08:15:44,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732204511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732144507, max timestamp=1733732144516Num uniques 859; 2024-12-09T08:15:44,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732144507, max timestamp=1733732144516Num uniques 859; 2024-12-09T08:15:44,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732144507, max timestamp=1733732144516Num uniques 859; 2024-12-09T08:15:44,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:44,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:44,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:44,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742160_1336 (size=48139) 2024-12-09T08:15:44,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732204547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732204547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732204554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732204555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T08:15:44,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T08:15:44,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:44,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:44,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:44,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732204621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732204661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732204661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732204661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732204670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,696 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:15:44,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T08:15:44,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:44,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:44,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T08:15:44,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732204831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732204871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732204871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732204871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732204881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:44,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/25b59846fa2540189980aebe42e6719e 2024-12-09T08:15:44,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T08:15:44,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:44,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:44,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:44,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:44,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/66f2216f65a0499b85eef988941feffa is 50, key is test_row_0/B:col10/1733732144470/Put/seqid=0 2024-12-09T08:15:44,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742161_1337 (size=12001) 2024-12-09T08:15:44,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/66f2216f65a0499b85eef988941feffa 2024-12-09T08:15:45,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/5e224bd71de844f2a8916e7746f27a88 is 50, key is test_row_0/C:col10/1733732144470/Put/seqid=0 2024-12-09T08:15:45,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742162_1338 (size=12001) 2024-12-09T08:15:45,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T08:15:45,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:45,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:45,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:45,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:45,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T08:15:45,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732205141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:45,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732205180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:45,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:45,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732205191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:45,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732205191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:45,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732205201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:45,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T08:15:45,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:45,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:45,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:45,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:45,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:45,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T08:15:45,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:45,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:45,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:45,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:45,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:45,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/5e224bd71de844f2a8916e7746f27a88 2024-12-09T08:15:45,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/25b59846fa2540189980aebe42e6719e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/25b59846fa2540189980aebe42e6719e 2024-12-09T08:15:45,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/25b59846fa2540189980aebe42e6719e, entries=250, sequenceid=15, filesize=47.0 K 2024-12-09T08:15:45,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/66f2216f65a0499b85eef988941feffa as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/66f2216f65a0499b85eef988941feffa 2024-12-09T08:15:45,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/66f2216f65a0499b85eef988941feffa, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T08:15:45,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/5e224bd71de844f2a8916e7746f27a88 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5e224bd71de844f2a8916e7746f27a88 2024-12-09T08:15:45,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5e224bd71de844f2a8916e7746f27a88, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T08:15:45,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5081394aff7face3f62aba376aebd75e in 954ms, sequenceid=15, compaction requested=false 2024-12-09T08:15:45,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:45,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T08:15:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:45,535 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-09T08:15:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:45,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094ee672b83c0b48468978608b850e3ac7_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732144545/Put/seqid=0 2024-12-09T08:15:45,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742163_1339 (size=12154) 2024-12-09T08:15:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T08:15:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:45,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732205701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732205701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732205701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:45,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732205706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:45,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:45,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732205711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:45,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732205811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:45,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:45,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732205821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:45,950 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094ee672b83c0b48468978608b850e3ac7_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094ee672b83c0b48468978608b850e3ac7_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:45,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/2528c7d115a54a36b3ca03d7cc3dc6f8, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:45,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/2528c7d115a54a36b3ca03d7cc3dc6f8 is 175, key is test_row_0/A:col10/1733732144545/Put/seqid=0 2024-12-09T08:15:45,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742164_1340 (size=30955) 2024-12-09T08:15:46,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732206021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:46,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732206031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:46,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732206331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:46,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732206341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:46,356 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/2528c7d115a54a36b3ca03d7cc3dc6f8 2024-12-09T08:15:46,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/befe3c9d588b4d0e8f8872b830034d55 is 50, key is test_row_0/B:col10/1733732144545/Put/seqid=0 2024-12-09T08:15:46,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742165_1341 (size=12001) 2024-12-09T08:15:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T08:15:46,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732206721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:46,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732206721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:46,728 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 5081394aff7face3f62aba376aebd75e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:46,728 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 5081394aff7face3f62aba376aebd75e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:46,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732206731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:46,736 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 5081394aff7face3f62aba376aebd75e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:46,766 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/befe3c9d588b4d0e8f8872b830034d55 2024-12-09T08:15:46,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/4d48196784374e8e947f0b05d6b63bc9 is 50, key is test_row_0/C:col10/1733732144545/Put/seqid=0 2024-12-09T08:15:46,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742166_1342 (size=12001) 2024-12-09T08:15:46,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732206851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732206851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:47,175 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/4d48196784374e8e947f0b05d6b63bc9 2024-12-09T08:15:47,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/2528c7d115a54a36b3ca03d7cc3dc6f8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2528c7d115a54a36b3ca03d7cc3dc6f8 2024-12-09T08:15:47,182 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2528c7d115a54a36b3ca03d7cc3dc6f8, entries=150, sequenceid=51, filesize=30.2 K 2024-12-09T08:15:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/befe3c9d588b4d0e8f8872b830034d55 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/befe3c9d588b4d0e8f8872b830034d55 2024-12-09T08:15:47,186 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/befe3c9d588b4d0e8f8872b830034d55, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T08:15:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/4d48196784374e8e947f0b05d6b63bc9 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/4d48196784374e8e947f0b05d6b63bc9 2024-12-09T08:15:47,190 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/4d48196784374e8e947f0b05d6b63bc9, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T08:15:47,191 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 5081394aff7face3f62aba376aebd75e in 1656ms, sequenceid=51, compaction requested=false 2024-12-09T08:15:47,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:47,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:47,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-09T08:15:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-09T08:15:47,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-09T08:15:47,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7260 sec 2024-12-09T08:15:47,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.7320 sec 2024-12-09T08:15:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:47,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T08:15:47,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:47,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:47,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:47,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:47,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:47,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:47,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093d7d27e00cc6417ebc0dc253b42aa71b_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732145655/Put/seqid=0 2024-12-09T08:15:47,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742167_1343 (size=14594) 2024-12-09T08:15:47,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732207939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:47,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732207941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:48,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732208051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732208051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:48,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732208271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:48,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732208271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:48,285 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:48,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093d7d27e00cc6417ebc0dc253b42aa71b_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093d7d27e00cc6417ebc0dc253b42aa71b_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:48,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/dc9d0a07bafb471f887b3a12bf4e8789, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:48,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/dc9d0a07bafb471f887b3a12bf4e8789 is 175, key is test_row_0/A:col10/1733732145655/Put/seqid=0 2024-12-09T08:15:48,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742168_1344 (size=39549) 2024-12-09T08:15:48,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732208581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:48,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732208581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T08:15:48,631 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:48,631 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-09T08:15:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T08:15:48,634 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:48,634 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:48,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:48,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/dc9d0a07bafb471f887b3a12bf4e8789 2024-12-09T08:15:48,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/d548155122614ada85450d1be577ea12 is 50, key is test_row_0/B:col10/1733732145655/Put/seqid=0 2024-12-09T08:15:48,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742169_1345 (size=12001) 2024-12-09T08:15:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T08:15:48,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732208741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:48,746 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 5081394aff7face3f62aba376aebd75e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:48,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732208751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:48,756 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 5081394aff7face3f62aba376aebd75e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:48,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732208760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:48,765 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 5081394aff7face3f62aba376aebd75e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:48,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T08:15:48,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:48,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:48,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T08:15:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:48,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:48,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T08:15:49,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T08:15:49,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:49,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:49,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:49,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:49,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:49,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:49,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732209091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:49,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732209101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:49,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:49,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:49,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/d548155122614ada85450d1be577ea12 2024-12-09T08:15:49,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/c800e20e6149435f9e1cc39f35cd0097 is 50, key is test_row_0/C:col10/1733732145655/Put/seqid=0 2024-12-09T08:15:49,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742170_1346 (size=12001) 2024-12-09T08:15:49,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/c800e20e6149435f9e1cc39f35cd0097 2024-12-09T08:15:49,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/dc9d0a07bafb471f887b3a12bf4e8789 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/dc9d0a07bafb471f887b3a12bf4e8789 2024-12-09T08:15:49,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/dc9d0a07bafb471f887b3a12bf4e8789, entries=200, sequenceid=63, filesize=38.6 K 2024-12-09T08:15:49,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/d548155122614ada85450d1be577ea12 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d548155122614ada85450d1be577ea12 2024-12-09T08:15:49,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d548155122614ada85450d1be577ea12, entries=150, sequenceid=63, filesize=11.7 K 2024-12-09T08:15:49,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/c800e20e6149435f9e1cc39f35cd0097 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/c800e20e6149435f9e1cc39f35cd0097 2024-12-09T08:15:49,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/c800e20e6149435f9e1cc39f35cd0097, entries=150, sequenceid=63, filesize=11.7 K 2024-12-09T08:15:49,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5081394aff7face3f62aba376aebd75e in 1279ms, sequenceid=63, compaction requested=true 2024-12-09T08:15:49,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:49,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:49,153 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:49,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:49,153 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:49,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:49,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:49,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:49,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:49,154 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:49,154 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:49,154 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/B is initiating minor compaction (all files) 2024-12-09T08:15:49,154 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/A is initiating minor compaction (all files) 2024-12-09T08:15:49,154 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/A in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:49,154 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/B in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:49,154 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/66f2216f65a0499b85eef988941feffa, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/befe3c9d588b4d0e8f8872b830034d55, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d548155122614ada85450d1be577ea12] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=35.2 K 2024-12-09T08:15:49,154 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/25b59846fa2540189980aebe42e6719e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2528c7d115a54a36b3ca03d7cc3dc6f8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/dc9d0a07bafb471f887b3a12bf4e8789] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=115.9 K 2024-12-09T08:15:49,154 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:49,154 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/25b59846fa2540189980aebe42e6719e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2528c7d115a54a36b3ca03d7cc3dc6f8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/dc9d0a07bafb471f887b3a12bf4e8789] 2024-12-09T08:15:49,155 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 66f2216f65a0499b85eef988941feffa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733732144470 2024-12-09T08:15:49,155 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25b59846fa2540189980aebe42e6719e, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733732144469 2024-12-09T08:15:49,155 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting befe3c9d588b4d0e8f8872b830034d55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733732144533 2024-12-09T08:15:49,155 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2528c7d115a54a36b3ca03d7cc3dc6f8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733732144533 2024-12-09T08:15:49,155 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting d548155122614ada85450d1be577ea12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733732145655 2024-12-09T08:15:49,155 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc9d0a07bafb471f887b3a12bf4e8789, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733732145655 2024-12-09T08:15:49,160 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:49,162 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412095a8395de04b442e8998ad9928bcba914_5081394aff7face3f62aba376aebd75e store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:49,162 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#B#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:49,162 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/301d7c7027814094aaece750125a9cde is 50, key is test_row_0/B:col10/1733732145655/Put/seqid=0 2024-12-09T08:15:49,163 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412095a8395de04b442e8998ad9928bcba914_5081394aff7face3f62aba376aebd75e, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:49,163 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095a8395de04b442e8998ad9928bcba914_5081394aff7face3f62aba376aebd75e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:49,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742171_1347 (size=12104) 2024-12-09T08:15:49,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742172_1348 (size=4469) 2024-12-09T08:15:49,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T08:15:49,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:49,245 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T08:15:49,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:49,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:49,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:49,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:49,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:49,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:49,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e52dd219fec34ade8f2fe7a3261b1e33_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732147939/Put/seqid=0 2024-12-09T08:15:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T08:15:49,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742173_1349 (size=12154) 2024-12-09T08:15:49,576 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/301d7c7027814094aaece750125a9cde as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/301d7c7027814094aaece750125a9cde 2024-12-09T08:15:49,577 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:15:49,578 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#A#compaction#299 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:49,579 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/bb360bf0511848179f9b0b2a6dbf34df is 175, key is test_row_0/A:col10/1733732145655/Put/seqid=0 2024-12-09T08:15:49,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742174_1350 (size=31058) 2024-12-09T08:15:49,590 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/B of 5081394aff7face3f62aba376aebd75e into 301d7c7027814094aaece750125a9cde(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:49,590 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:49,590 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/B, priority=13, startTime=1733732149153; duration=0sec 2024-12-09T08:15:49,590 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:49,590 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:B 2024-12-09T08:15:49,590 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:49,591 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:49,591 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/C is initiating minor compaction (all files) 2024-12-09T08:15:49,591 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/C in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:49,592 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5e224bd71de844f2a8916e7746f27a88, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/4d48196784374e8e947f0b05d6b63bc9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/c800e20e6149435f9e1cc39f35cd0097] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=35.2 K 2024-12-09T08:15:49,592 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e224bd71de844f2a8916e7746f27a88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733732144470 2024-12-09T08:15:49,593 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d48196784374e8e947f0b05d6b63bc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733732144533 2024-12-09T08:15:49,593 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting c800e20e6149435f9e1cc39f35cd0097, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733732145655 2024-12-09T08:15:49,595 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/bb360bf0511848179f9b0b2a6dbf34df as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/bb360bf0511848179f9b0b2a6dbf34df 2024-12-09T08:15:49,600 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/A of 5081394aff7face3f62aba376aebd75e into bb360bf0511848179f9b0b2a6dbf34df(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:49,600 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:49,600 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/A, priority=13, startTime=1733732149153; duration=0sec 2024-12-09T08:15:49,600 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:49,600 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:A 2024-12-09T08:15:49,603 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#C#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:49,604 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/bb11c0ea99a94f94acda22c1d23f7a5a is 50, key is test_row_0/C:col10/1733732145655/Put/seqid=0 2024-12-09T08:15:49,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742175_1351 (size=12104) 2024-12-09T08:15:49,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:49,672 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e52dd219fec34ade8f2fe7a3261b1e33_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e52dd219fec34ade8f2fe7a3261b1e33_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:49,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/02478f2ebabf4c459d710eb5e760a70a, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:49,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/02478f2ebabf4c459d710eb5e760a70a is 175, key is test_row_0/A:col10/1733732147939/Put/seqid=0 2024-12-09T08:15:49,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742176_1352 (size=30955) 2024-12-09T08:15:49,678 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/02478f2ebabf4c459d710eb5e760a70a 2024-12-09T08:15:49,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/7d8c6ee5429c46a4b0a02e82af4f781b is 50, key is test_row_0/B:col10/1733732147939/Put/seqid=0 2024-12-09T08:15:49,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742177_1353 (size=12001) 2024-12-09T08:15:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T08:15:50,012 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/bb11c0ea99a94f94acda22c1d23f7a5a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/bb11c0ea99a94f94acda22c1d23f7a5a 2024-12-09T08:15:50,017 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/C of 5081394aff7face3f62aba376aebd75e into bb11c0ea99a94f94acda22c1d23f7a5a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:50,017 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:50,018 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/C, priority=13, startTime=1733732149154; duration=0sec 2024-12-09T08:15:50,018 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:50,019 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:C 2024-12-09T08:15:50,095 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/7d8c6ee5429c46a4b0a02e82af4f781b 2024-12-09T08:15:50,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/b6ef12ce06014edd89565d53993bf029 is 50, key is test_row_0/C:col10/1733732147939/Put/seqid=0 2024-12-09T08:15:50,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742178_1354 (size=12001) 2024-12-09T08:15:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:50,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:50,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732210135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:50,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732210137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:50,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732210251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:50,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732210251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:50,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732210461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:50,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732210461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:50,505 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/b6ef12ce06014edd89565d53993bf029 2024-12-09T08:15:50,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/02478f2ebabf4c459d710eb5e760a70a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/02478f2ebabf4c459d710eb5e760a70a 2024-12-09T08:15:50,512 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/02478f2ebabf4c459d710eb5e760a70a, entries=150, sequenceid=83, filesize=30.2 K 2024-12-09T08:15:50,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/7d8c6ee5429c46a4b0a02e82af4f781b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/7d8c6ee5429c46a4b0a02e82af4f781b 2024-12-09T08:15:50,517 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/7d8c6ee5429c46a4b0a02e82af4f781b, entries=150, sequenceid=83, filesize=11.7 K 2024-12-09T08:15:50,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/b6ef12ce06014edd89565d53993bf029 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/b6ef12ce06014edd89565d53993bf029 2024-12-09T08:15:50,521 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/b6ef12ce06014edd89565d53993bf029, entries=150, sequenceid=83, filesize=11.7 K 2024-12-09T08:15:50,521 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5081394aff7face3f62aba376aebd75e in 1276ms, sequenceid=83, compaction requested=false 2024-12-09T08:15:50,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:50,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:50,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-09T08:15:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-09T08:15:50,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-09T08:15:50,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8890 sec 2024-12-09T08:15:50,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.8940 sec 2024-12-09T08:15:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:50,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T08:15:50,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:50,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:50,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:50,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:50,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:50,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T08:15:50,782 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:50,782 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-09T08:15:50,784 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T08:15:50,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f3b306ed5d2d4309952016a995aba728_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732150118/Put/seqid=0 2024-12-09T08:15:50,784 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:50,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:50,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742179_1355 (size=12154) 2024-12-09T08:15:50,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732210846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732210846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T08:15:50,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T08:15:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:50,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:50,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:50,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:50,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:50,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732210961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:50,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732210961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:51,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T08:15:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:51,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T08:15:51,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732211172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:51,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732211172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:51,190 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:51,194 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f3b306ed5d2d4309952016a995aba728_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f3b306ed5d2d4309952016a995aba728_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:51,197 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/162295660d9846d5b0c61d58811bf395, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:51,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/162295660d9846d5b0c61d58811bf395 is 175, key is test_row_0/A:col10/1733732150118/Put/seqid=0 2024-12-09T08:15:51,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742180_1356 (size=30955) 2024-12-09T08:15:51,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T08:15:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:51,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T08:15:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:51,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T08:15:51,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732211481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:51,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732211491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:51,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T08:15:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:51,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,607 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/162295660d9846d5b0c61d58811bf395 2024-12-09T08:15:51,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/b603c39c4fe6440da595cb4948b78ade is 50, key is test_row_0/B:col10/1733732150118/Put/seqid=0 2024-12-09T08:15:51,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742181_1357 (size=12001) 2024-12-09T08:15:51,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T08:15:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:51,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T08:15:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:51,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T08:15:51,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732211991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:52,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732212001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:52,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T08:15:52,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:52,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:52,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:52,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:52,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:52,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/b603c39c4fe6440da595cb4948b78ade 2024-12-09T08:15:52,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/edb8f7dd32714564a0698ef120495700 is 50, key is test_row_0/C:col10/1733732150118/Put/seqid=0 2024-12-09T08:15:52,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742182_1358 (size=12001) 2024-12-09T08:15:52,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T08:15:52,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:52,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:52,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:52,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:52,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:52,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T08:15:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:52,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:52,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/edb8f7dd32714564a0698ef120495700 2024-12-09T08:15:52,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/162295660d9846d5b0c61d58811bf395 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/162295660d9846d5b0c61d58811bf395 2024-12-09T08:15:52,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/162295660d9846d5b0c61d58811bf395, entries=150, sequenceid=100, filesize=30.2 K 2024-12-09T08:15:52,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/b603c39c4fe6440da595cb4948b78ade as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/b603c39c4fe6440da595cb4948b78ade 2024-12-09T08:15:52,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/b603c39c4fe6440da595cb4948b78ade, entries=150, sequenceid=100, filesize=11.7 K 2024-12-09T08:15:52,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/edb8f7dd32714564a0698ef120495700 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/edb8f7dd32714564a0698ef120495700 2024-12-09T08:15:52,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/edb8f7dd32714564a0698ef120495700, entries=150, sequenceid=100, filesize=11.7 K 2024-12-09T08:15:52,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 5081394aff7face3f62aba376aebd75e in 1663ms, sequenceid=100, compaction requested=true 2024-12-09T08:15:52,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:52,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:52,441 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:52,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:52,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:52,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:52,442 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:52,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:52,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:52,442 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:52,442 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/A is initiating minor compaction (all files) 2024-12-09T08:15:52,442 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/A in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:52,442 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/bb360bf0511848179f9b0b2a6dbf34df, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/02478f2ebabf4c459d710eb5e760a70a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/162295660d9846d5b0c61d58811bf395] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=90.8 K 2024-12-09T08:15:52,443 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:52,443 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/bb360bf0511848179f9b0b2a6dbf34df, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/02478f2ebabf4c459d710eb5e760a70a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/162295660d9846d5b0c61d58811bf395] 2024-12-09T08:15:52,443 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:52,443 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/B is initiating minor compaction (all files) 2024-12-09T08:15:52,443 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/B in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:52,443 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/301d7c7027814094aaece750125a9cde, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/7d8c6ee5429c46a4b0a02e82af4f781b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/b603c39c4fe6440da595cb4948b78ade] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=35.3 K 2024-12-09T08:15:52,443 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb360bf0511848179f9b0b2a6dbf34df, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733732145655 2024-12-09T08:15:52,443 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 301d7c7027814094aaece750125a9cde, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733732145655 2024-12-09T08:15:52,443 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02478f2ebabf4c459d710eb5e760a70a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733732147924 2024-12-09T08:15:52,443 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d8c6ee5429c46a4b0a02e82af4f781b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733732147924 2024-12-09T08:15:52,444 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting b603c39c4fe6440da595cb4948b78ade, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733732150118 2024-12-09T08:15:52,444 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 162295660d9846d5b0c61d58811bf395, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733732150118 2024-12-09T08:15:52,449 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:52,450 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#B#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:52,450 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/96213dab89f94b58a6b514ef0feb982d is 50, key is test_row_0/B:col10/1733732150118/Put/seqid=0 2024-12-09T08:15:52,452 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209a1cad2f4ad5f430fbf02611e1bc105e9_5081394aff7face3f62aba376aebd75e store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:52,453 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209a1cad2f4ad5f430fbf02611e1bc105e9_5081394aff7face3f62aba376aebd75e, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:52,453 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a1cad2f4ad5f430fbf02611e1bc105e9_5081394aff7face3f62aba376aebd75e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:52,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742183_1359 (size=12207) 2024-12-09T08:15:52,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742184_1360 (size=4469) 2024-12-09T08:15:52,461 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/96213dab89f94b58a6b514ef0feb982d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/96213dab89f94b58a6b514ef0feb982d 2024-12-09T08:15:52,466 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/B of 5081394aff7face3f62aba376aebd75e into 96213dab89f94b58a6b514ef0feb982d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:52,466 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:52,466 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/B, priority=13, startTime=1733732152441; duration=0sec 2024-12-09T08:15:52,467 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:52,467 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:B 2024-12-09T08:15:52,467 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:52,467 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:52,467 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/C is initiating minor compaction (all files) 2024-12-09T08:15:52,467 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/C in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:52,468 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/bb11c0ea99a94f94acda22c1d23f7a5a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/b6ef12ce06014edd89565d53993bf029, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/edb8f7dd32714564a0698ef120495700] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=35.3 K 2024-12-09T08:15:52,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T08:15:52,468 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting bb11c0ea99a94f94acda22c1d23f7a5a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733732145655 2024-12-09T08:15:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:52,468 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T08:15:52,468 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting b6ef12ce06014edd89565d53993bf029, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733732147924 2024-12-09T08:15:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:52,468 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting edb8f7dd32714564a0698ef120495700, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733732150118 2024-12-09T08:15:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:52,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097d59194c1bde4776b9c763e6faff77bc_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732150845/Put/seqid=0 2024-12-09T08:15:52,482 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:52,482 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/c735aa743e6f4b9cb74758f6c4d0f968 is 50, key is test_row_0/C:col10/1733732150118/Put/seqid=0 2024-12-09T08:15:52,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742185_1361 (size=12154) 2024-12-09T08:15:52,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742186_1362 (size=12207) 2024-12-09T08:15:52,503 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/c735aa743e6f4b9cb74758f6c4d0f968 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/c735aa743e6f4b9cb74758f6c4d0f968 2024-12-09T08:15:52,508 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/C of 5081394aff7face3f62aba376aebd75e into c735aa743e6f4b9cb74758f6c4d0f968(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:52,508 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:52,508 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/C, priority=13, startTime=1733732152442; duration=0sec 2024-12-09T08:15:52,508 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:52,508 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:C 2024-12-09T08:15:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:52,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732212799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:52,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:52,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732212800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732212803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:52,859 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#A#compaction#309 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:52,859 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/bf5369f397434a20a7b071e55b254972 is 175, key is test_row_0/A:col10/1733732150118/Put/seqid=0 2024-12-09T08:15:52,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742187_1363 (size=31161) 2024-12-09T08:15:52,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:52,894 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097d59194c1bde4776b9c763e6faff77bc_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097d59194c1bde4776b9c763e6faff77bc_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/aea477cb13b74b53bc79af9843afd57e, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/aea477cb13b74b53bc79af9843afd57e is 175, key is test_row_0/A:col10/1733732150845/Put/seqid=0 2024-12-09T08:15:52,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742188_1364 (size=30955) 2024-12-09T08:15:52,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732212911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732212922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T08:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732212922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:53,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732213001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:53,006 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in 5081394aff7face3f62aba376aebd75e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:53,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732213021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:53,027 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 5081394aff7face3f62aba376aebd75e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:53,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732213131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:53,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732213141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732213141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:53,267 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/bf5369f397434a20a7b071e55b254972 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/bf5369f397434a20a7b071e55b254972 2024-12-09T08:15:53,271 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/A of 5081394aff7face3f62aba376aebd75e into bf5369f397434a20a7b071e55b254972(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:53,271 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:53,271 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/A, priority=13, startTime=1733732152441; duration=0sec 2024-12-09T08:15:53,271 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:53,271 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:A 2024-12-09T08:15:53,301 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/aea477cb13b74b53bc79af9843afd57e 2024-12-09T08:15:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/83e291cdf4ab41eeb0e21013fbc4de69 is 50, key is test_row_0/B:col10/1733732150845/Put/seqid=0 2024-12-09T08:15:53,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742189_1365 (size=12001) 2024-12-09T08:15:53,310 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/83e291cdf4ab41eeb0e21013fbc4de69 2024-12-09T08:15:53,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/12e93a5a430343568d6237ee3e4aeecd is 50, key is test_row_0/C:col10/1733732150845/Put/seqid=0 2024-12-09T08:15:53,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742190_1366 (size=12001) 2024-12-09T08:15:53,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:53,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732213440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:53,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:53,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732213451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:53,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:53,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732213451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:53,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:53,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:53,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:53,721 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/12e93a5a430343568d6237ee3e4aeecd 2024-12-09T08:15:53,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/aea477cb13b74b53bc79af9843afd57e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/aea477cb13b74b53bc79af9843afd57e 2024-12-09T08:15:53,727 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/aea477cb13b74b53bc79af9843afd57e, entries=150, sequenceid=120, filesize=30.2 K 2024-12-09T08:15:53,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/83e291cdf4ab41eeb0e21013fbc4de69 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/83e291cdf4ab41eeb0e21013fbc4de69 2024-12-09T08:15:53,731 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/83e291cdf4ab41eeb0e21013fbc4de69, entries=150, sequenceid=120, filesize=11.7 K 2024-12-09T08:15:53,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/12e93a5a430343568d6237ee3e4aeecd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/12e93a5a430343568d6237ee3e4aeecd 2024-12-09T08:15:53,735 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/12e93a5a430343568d6237ee3e4aeecd, entries=150, sequenceid=120, filesize=11.7 K 2024-12-09T08:15:53,736 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5081394aff7face3f62aba376aebd75e in 1267ms, sequenceid=120, compaction requested=false 2024-12-09T08:15:53,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:53,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:53,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-09T08:15:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-09T08:15:53,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-09T08:15:53,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9530 sec 2024-12-09T08:15:53,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.9570 sec 2024-12-09T08:15:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:53,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T08:15:53,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:53,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:53,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:53,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:53,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:53,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:53,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092eaeb32e714a4dba90ff9a99eeeff06b_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732152797/Put/seqid=0 2024-12-09T08:15:53,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742191_1367 (size=14744) 2024-12-09T08:15:54,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732214003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732214005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732214007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732214121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732214121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732214121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732214331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732214331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732214341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,377 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:54,380 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092eaeb32e714a4dba90ff9a99eeeff06b_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092eaeb32e714a4dba90ff9a99eeeff06b_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:54,381 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/30ab2dd6281b42208821632746072ae4, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:54,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/30ab2dd6281b42208821632746072ae4 is 175, key is test_row_0/A:col10/1733732152797/Put/seqid=0 2024-12-09T08:15:54,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742192_1368 (size=39699) 2024-12-09T08:15:54,386 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/30ab2dd6281b42208821632746072ae4 2024-12-09T08:15:54,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/b9285353a6ca468e9b77150f124a90d5 is 50, key is test_row_0/B:col10/1733732152797/Put/seqid=0 2024-12-09T08:15:54,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742193_1369 (size=12101) 2024-12-09T08:15:54,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/b9285353a6ca468e9b77150f124a90d5 2024-12-09T08:15:54,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/4a3c88a90eb340d7b1ec6033a9f078ac is 50, key is test_row_0/C:col10/1733732152797/Put/seqid=0 2024-12-09T08:15:54,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742194_1370 (size=12101) 2024-12-09T08:15:54,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/4a3c88a90eb340d7b1ec6033a9f078ac 2024-12-09T08:15:54,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/30ab2dd6281b42208821632746072ae4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/30ab2dd6281b42208821632746072ae4 2024-12-09T08:15:54,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/30ab2dd6281b42208821632746072ae4, entries=200, sequenceid=136, filesize=38.8 K 2024-12-09T08:15:54,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/b9285353a6ca468e9b77150f124a90d5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/b9285353a6ca468e9b77150f124a90d5 2024-12-09T08:15:54,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/b9285353a6ca468e9b77150f124a90d5, entries=150, sequenceid=136, filesize=11.8 K 2024-12-09T08:15:54,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/4a3c88a90eb340d7b1ec6033a9f078ac as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/4a3c88a90eb340d7b1ec6033a9f078ac 2024-12-09T08:15:54,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/4a3c88a90eb340d7b1ec6033a9f078ac, entries=150, sequenceid=136, filesize=11.8 K 2024-12-09T08:15:54,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5081394aff7face3f62aba376aebd75e in 509ms, sequenceid=136, compaction requested=true 2024-12-09T08:15:54,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:54,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:54,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:54,464 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:54,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:54,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:54,464 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:54,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:54,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:54,464 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:54,464 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/A is initiating minor compaction (all files) 2024-12-09T08:15:54,465 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/A in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:54,465 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/bf5369f397434a20a7b071e55b254972, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/aea477cb13b74b53bc79af9843afd57e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/30ab2dd6281b42208821632746072ae4] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=99.4 K 2024-12-09T08:15:54,465 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:54,465 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/bf5369f397434a20a7b071e55b254972, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/aea477cb13b74b53bc79af9843afd57e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/30ab2dd6281b42208821632746072ae4] 2024-12-09T08:15:54,465 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:54,465 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/B is initiating minor compaction (all files) 2024-12-09T08:15:54,465 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/B in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:54,465 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/96213dab89f94b58a6b514ef0feb982d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/83e291cdf4ab41eeb0e21013fbc4de69, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/b9285353a6ca468e9b77150f124a90d5] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=35.5 K 2024-12-09T08:15:54,466 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 96213dab89f94b58a6b514ef0feb982d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733732150118 2024-12-09T08:15:54,466 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf5369f397434a20a7b071e55b254972, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733732150118 2024-12-09T08:15:54,466 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e291cdf4ab41eeb0e21013fbc4de69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733732150812 2024-12-09T08:15:54,466 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting aea477cb13b74b53bc79af9843afd57e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733732150812 2024-12-09T08:15:54,466 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting b9285353a6ca468e9b77150f124a90d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733732152785 2024-12-09T08:15:54,466 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30ab2dd6281b42208821632746072ae4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733732152785 2024-12-09T08:15:54,472 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:54,473 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:54,474 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/33c4098a2400441fb776a008465e6952 is 50, key is test_row_0/B:col10/1733732152797/Put/seqid=0 2024-12-09T08:15:54,478 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209d392d24fedf540dab1547e1baafa7829_5081394aff7face3f62aba376aebd75e store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:54,480 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209d392d24fedf540dab1547e1baafa7829_5081394aff7face3f62aba376aebd75e, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:54,480 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d392d24fedf540dab1547e1baafa7829_5081394aff7face3f62aba376aebd75e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:54,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742196_1372 (size=4469) 2024-12-09T08:15:54,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742195_1371 (size=12409) 2024-12-09T08:15:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:54,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T08:15:54,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:54,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:54,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:54,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:54,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:54,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:54,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096ba1090985a14226aa8fd90ff7a86cab_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732153979/Put/seqid=0 2024-12-09T08:15:54,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742197_1373 (size=14794) 2024-12-09T08:15:54,656 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:54,659 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096ba1090985a14226aa8fd90ff7a86cab_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096ba1090985a14226aa8fd90ff7a86cab_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:54,659 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/595a4afba1d743e79dd57eca03e26420, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:54,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/595a4afba1d743e79dd57eca03e26420 is 175, key is test_row_0/A:col10/1733732153979/Put/seqid=0 2024-12-09T08:15:54,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742198_1374 (size=39749) 2024-12-09T08:15:54,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732214672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732214681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732214682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732214790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732214791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732214801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:54,894 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:54,895 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/000e4ee4a80d4918bae11fe59fff2bdd is 175, key is test_row_0/A:col10/1733732152797/Put/seqid=0 2024-12-09T08:15:54,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742199_1375 (size=31363) 2024-12-09T08:15:54,902 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/33c4098a2400441fb776a008465e6952 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/33c4098a2400441fb776a008465e6952 2024-12-09T08:15:54,904 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/000e4ee4a80d4918bae11fe59fff2bdd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/000e4ee4a80d4918bae11fe59fff2bdd 2024-12-09T08:15:54,907 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/B of 5081394aff7face3f62aba376aebd75e into 33c4098a2400441fb776a008465e6952(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:54,907 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:54,907 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/B, priority=13, startTime=1733732154464; duration=0sec 2024-12-09T08:15:54,907 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:54,907 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:B 2024-12-09T08:15:54,907 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:54,909 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/A of 5081394aff7face3f62aba376aebd75e into 000e4ee4a80d4918bae11fe59fff2bdd(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:54,909 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:54,909 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/A, priority=13, startTime=1733732154464; duration=0sec 2024-12-09T08:15:54,909 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:54,909 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:A 2024-12-09T08:15:54,910 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:54,910 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/C is initiating minor compaction (all files) 2024-12-09T08:15:54,910 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/C in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:54,910 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/c735aa743e6f4b9cb74758f6c4d0f968, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/12e93a5a430343568d6237ee3e4aeecd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/4a3c88a90eb340d7b1ec6033a9f078ac] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=35.5 K 2024-12-09T08:15:54,910 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting c735aa743e6f4b9cb74758f6c4d0f968, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733732150118 2024-12-09T08:15:54,911 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e93a5a430343568d6237ee3e4aeecd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733732150812 2024-12-09T08:15:54,911 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a3c88a90eb340d7b1ec6033a9f078ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733732152785 2024-12-09T08:15:54,918 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:54,918 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/429f59ca0cde4a72b47a30e40836897e is 50, key is test_row_0/C:col10/1733732152797/Put/seqid=0 2024-12-09T08:15:54,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742200_1376 (size=12409) 2024-12-09T08:15:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T08:15:54,951 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:54,952 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-09T08:15:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T08:15:54,953 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:54,954 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:54,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:55,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732215000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732215001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732215011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732215031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,040 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in 5081394aff7face3f62aba376aebd75e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732215051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T08:15:55,060 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 5081394aff7face3f62aba376aebd75e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:55,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,063 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/595a4afba1d743e79dd57eca03e26420 2024-12-09T08:15:55,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/37dbe69015d249b0b843f2f288f9231b is 50, key is test_row_0/B:col10/1733732153979/Put/seqid=0 2024-12-09T08:15:55,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742201_1377 (size=12151) 2024-12-09T08:15:55,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/37dbe69015d249b0b843f2f288f9231b 2024-12-09T08:15:55,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/47a8b9d8a4184b5bae006413355b96cf is 50, key is test_row_0/C:col10/1733732153979/Put/seqid=0 2024-12-09T08:15:55,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742202_1378 (size=12151) 2024-12-09T08:15:55,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T08:15:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:55,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:55,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T08:15:55,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:55,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:55,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:55,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:55,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T08:15:55,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732215311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732215321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,330 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/429f59ca0cde4a72b47a30e40836897e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/429f59ca0cde4a72b47a30e40836897e 2024-12-09T08:15:55,333 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/C of 5081394aff7face3f62aba376aebd75e into 429f59ca0cde4a72b47a30e40836897e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:55,333 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:55,333 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/C, priority=13, startTime=1733732154464; duration=0sec 2024-12-09T08:15:55,334 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:55,334 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:C 2024-12-09T08:15:55,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732215331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T08:15:55,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:55,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:55,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:55,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:55,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:55,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/47a8b9d8a4184b5bae006413355b96cf 2024-12-09T08:15:55,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/595a4afba1d743e79dd57eca03e26420 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/595a4afba1d743e79dd57eca03e26420 2024-12-09T08:15:55,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/595a4afba1d743e79dd57eca03e26420, entries=200, sequenceid=155, filesize=38.8 K 2024-12-09T08:15:55,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/37dbe69015d249b0b843f2f288f9231b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/37dbe69015d249b0b843f2f288f9231b 2024-12-09T08:15:55,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/37dbe69015d249b0b843f2f288f9231b, entries=150, sequenceid=155, filesize=11.9 K 2024-12-09T08:15:55,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/47a8b9d8a4184b5bae006413355b96cf as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/47a8b9d8a4184b5bae006413355b96cf 2024-12-09T08:15:55,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/47a8b9d8a4184b5bae006413355b96cf, entries=150, sequenceid=155, filesize=11.9 K 2024-12-09T08:15:55,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 5081394aff7face3f62aba376aebd75e in 853ms, sequenceid=155, compaction requested=false 2024-12-09T08:15:55,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:55,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T08:15:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:55,565 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T08:15:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:55,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120988997535cb974af3ae8143a382fabbbe_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732154679/Put/seqid=0 2024-12-09T08:15:55,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742203_1379 (size=12304) 2024-12-09T08:15:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T08:15:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:55,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:55,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732215857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732215858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732215858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:55,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732215971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732215971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732215971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:55,977 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120988997535cb974af3ae8143a382fabbbe_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120988997535cb974af3ae8143a382fabbbe_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:55,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/64980ee2ea5e49378c9e012991c7b025, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:55,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/64980ee2ea5e49378c9e012991c7b025 is 175, key is test_row_0/A:col10/1733732154679/Put/seqid=0 2024-12-09T08:15:55,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742204_1380 (size=31105) 2024-12-09T08:15:55,982 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/64980ee2ea5e49378c9e012991c7b025 2024-12-09T08:15:55,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/3136d8da636c43cd8fc1462395d04840 is 50, key is test_row_0/B:col10/1733732154679/Put/seqid=0 2024-12-09T08:15:55,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742205_1381 (size=12151) 2024-12-09T08:15:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T08:15:56,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732216181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:56,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732216181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:56,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732216181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:56,390 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/3136d8da636c43cd8fc1462395d04840 2024-12-09T08:15:56,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/2e26d47b2dd24170bafcecd76a2b43af is 50, key is test_row_0/C:col10/1733732154679/Put/seqid=0 2024-12-09T08:15:56,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742206_1382 (size=12151) 2024-12-09T08:15:56,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732216490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:56,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732216490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:56,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732216491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:56,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:56,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:56,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:56,799 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/2e26d47b2dd24170bafcecd76a2b43af 2024-12-09T08:15:56,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/64980ee2ea5e49378c9e012991c7b025 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/64980ee2ea5e49378c9e012991c7b025 2024-12-09T08:15:56,806 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/64980ee2ea5e49378c9e012991c7b025, entries=150, sequenceid=172, filesize=30.4 K 2024-12-09T08:15:56,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/3136d8da636c43cd8fc1462395d04840 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/3136d8da636c43cd8fc1462395d04840 2024-12-09T08:15:56,809 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/3136d8da636c43cd8fc1462395d04840, entries=150, sequenceid=172, filesize=11.9 K 2024-12-09T08:15:56,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/2e26d47b2dd24170bafcecd76a2b43af as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/2e26d47b2dd24170bafcecd76a2b43af 2024-12-09T08:15:56,815 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/2e26d47b2dd24170bafcecd76a2b43af, entries=150, sequenceid=172, filesize=11.9 K 2024-12-09T08:15:56,816 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5081394aff7face3f62aba376aebd75e in 1251ms, sequenceid=172, compaction requested=true 2024-12-09T08:15:56,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:56,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:56,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-09T08:15:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-09T08:15:56,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-09T08:15:56,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8630 sec 2024-12-09T08:15:56,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8670 sec 2024-12-09T08:15:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:57,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T08:15:57,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:57,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:57,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:57,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:57,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:57,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:57,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c7a016b2672e493aa948bc8d4df58667_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732157008/Put/seqid=0 2024-12-09T08:15:57,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742207_1383 (size=14794) 2024-12-09T08:15:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732217029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732217030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:57,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732217033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T08:15:57,101 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:57,102 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-09T08:15:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T08:15:57,103 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:57,104 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:57,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:57,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732217141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:57,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732217142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:57,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732217151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T08:15:57,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T08:15:57,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:57,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732217350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:57,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732217351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:57,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732217361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:57,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T08:15:57,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:57,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T08:15:57,424 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:57,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c7a016b2672e493aa948bc8d4df58667_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c7a016b2672e493aa948bc8d4df58667_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:57,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/55c47d7aabd5471aaea8ab700bc49662, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:57,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/55c47d7aabd5471aaea8ab700bc49662 is 175, key is test_row_0/A:col10/1733732157008/Put/seqid=0 2024-12-09T08:15:57,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742208_1384 (size=39749) 2024-12-09T08:15:57,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T08:15:57,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732217671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732217671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:57,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732217671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:57,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T08:15:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T08:15:57,844 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/55c47d7aabd5471aaea8ab700bc49662 2024-12-09T08:15:57,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/dc1aac7136004ebead9effdb51dcb0c4 is 50, key is test_row_0/B:col10/1733732157008/Put/seqid=0 2024-12-09T08:15:57,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T08:15:57,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:57,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:15:57,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742209_1385 (size=12151) 2024-12-09T08:15:57,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/dc1aac7136004ebead9effdb51dcb0c4 2024-12-09T08:15:57,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/ca1874c891314b1fb65aad002e2642db is 50, key is test_row_0/C:col10/1733732157008/Put/seqid=0 2024-12-09T08:15:57,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742210_1386 (size=12151) 2024-12-09T08:15:57,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/ca1874c891314b1fb65aad002e2642db 2024-12-09T08:15:57,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/55c47d7aabd5471aaea8ab700bc49662 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/55c47d7aabd5471aaea8ab700bc49662 2024-12-09T08:15:57,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/55c47d7aabd5471aaea8ab700bc49662, entries=200, sequenceid=193, filesize=38.8 K 2024-12-09T08:15:57,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/dc1aac7136004ebead9effdb51dcb0c4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/dc1aac7136004ebead9effdb51dcb0c4 2024-12-09T08:15:57,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/dc1aac7136004ebead9effdb51dcb0c4, entries=150, sequenceid=193, filesize=11.9 K 2024-12-09T08:15:57,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/ca1874c891314b1fb65aad002e2642db as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/ca1874c891314b1fb65aad002e2642db 2024-12-09T08:15:57,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/ca1874c891314b1fb65aad002e2642db, entries=150, sequenceid=193, filesize=11.9 K 2024-12-09T08:15:57,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 5081394aff7face3f62aba376aebd75e in 904ms, sequenceid=193, compaction requested=true 2024-12-09T08:15:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:57,913 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:15:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:57,913 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:15:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:57,914 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:15:57,914 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:15:57,914 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/A is initiating minor compaction (all files) 2024-12-09T08:15:57,914 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/B is initiating minor compaction (all files) 2024-12-09T08:15:57,914 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/A in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,914 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/B in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,914 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/000e4ee4a80d4918bae11fe59fff2bdd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/595a4afba1d743e79dd57eca03e26420, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/64980ee2ea5e49378c9e012991c7b025, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/55c47d7aabd5471aaea8ab700bc49662] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=138.6 K 2024-12-09T08:15:57,914 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/33c4098a2400441fb776a008465e6952, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/37dbe69015d249b0b843f2f288f9231b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/3136d8da636c43cd8fc1462395d04840, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/dc1aac7136004ebead9effdb51dcb0c4] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=47.7 K 2024-12-09T08:15:57,914 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,914 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/000e4ee4a80d4918bae11fe59fff2bdd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/595a4afba1d743e79dd57eca03e26420, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/64980ee2ea5e49378c9e012991c7b025, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/55c47d7aabd5471aaea8ab700bc49662] 2024-12-09T08:15:57,914 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c4098a2400441fb776a008465e6952, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733732152785 2024-12-09T08:15:57,914 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 000e4ee4a80d4918bae11fe59fff2bdd, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733732152785 2024-12-09T08:15:57,914 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 37dbe69015d249b0b843f2f288f9231b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733732153979 2024-12-09T08:15:57,914 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 595a4afba1d743e79dd57eca03e26420, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733732153979 2024-12-09T08:15:57,915 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3136d8da636c43cd8fc1462395d04840, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733732154665 2024-12-09T08:15:57,915 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64980ee2ea5e49378c9e012991c7b025, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733732154665 2024-12-09T08:15:57,915 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting dc1aac7136004ebead9effdb51dcb0c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733732157007 2024-12-09T08:15:57,915 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55c47d7aabd5471aaea8ab700bc49662, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733732155855 2024-12-09T08:15:57,923 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#B#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:57,924 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/685fe56202754ad4a5e994e03db1c1c3 is 50, key is test_row_0/B:col10/1733732157008/Put/seqid=0 2024-12-09T08:15:57,929 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:57,940 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209106c80a54cd042139bbb0ee56305dcbe_5081394aff7face3f62aba376aebd75e store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:57,942 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209106c80a54cd042139bbb0ee56305dcbe_5081394aff7face3f62aba376aebd75e, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:57,943 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209106c80a54cd042139bbb0ee56305dcbe_5081394aff7face3f62aba376aebd75e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:57,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742211_1387 (size=12595) 2024-12-09T08:15:57,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742212_1388 (size=4469) 2024-12-09T08:15:57,966 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#A#compaction#330 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:57,966 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/5b8b76d3142d4b1c9e11dd3a2cab1884 is 175, key is test_row_0/A:col10/1733732157008/Put/seqid=0 2024-12-09T08:15:57,971 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/685fe56202754ad4a5e994e03db1c1c3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/685fe56202754ad4a5e994e03db1c1c3 2024-12-09T08:15:57,974 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5081394aff7face3f62aba376aebd75e/B of 5081394aff7face3f62aba376aebd75e into 685fe56202754ad4a5e994e03db1c1c3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:57,974 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:57,974 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/B, priority=12, startTime=1733732157913; duration=0sec 2024-12-09T08:15:57,974 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:57,974 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:B 2024-12-09T08:15:57,975 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:15:57,975 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:15:57,975 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/C is initiating minor compaction (all files) 2024-12-09T08:15:57,975 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/C in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:57,975 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/429f59ca0cde4a72b47a30e40836897e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/47a8b9d8a4184b5bae006413355b96cf, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/2e26d47b2dd24170bafcecd76a2b43af, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/ca1874c891314b1fb65aad002e2642db] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=47.7 K 2024-12-09T08:15:57,976 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 429f59ca0cde4a72b47a30e40836897e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733732152785 2024-12-09T08:15:57,976 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 47a8b9d8a4184b5bae006413355b96cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733732153979 2024-12-09T08:15:57,976 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e26d47b2dd24170bafcecd76a2b43af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733732154665 2024-12-09T08:15:57,977 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ca1874c891314b1fb65aad002e2642db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733732157007 2024-12-09T08:15:57,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742213_1389 (size=31549) 2024-12-09T08:15:57,989 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#C#compaction#331 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:57,990 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/30510b30a730457f8c31cbab370289cb is 50, key is test_row_0/C:col10/1733732157008/Put/seqid=0 2024-12-09T08:15:58,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742214_1390 (size=12595) 2024-12-09T08:15:58,017 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/30510b30a730457f8c31cbab370289cb as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/30510b30a730457f8c31cbab370289cb 2024-12-09T08:15:58,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T08:15:58,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:58,022 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:15:58,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:58,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:58,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:58,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:58,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:58,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:58,026 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5081394aff7face3f62aba376aebd75e/C of 5081394aff7face3f62aba376aebd75e into 30510b30a730457f8c31cbab370289cb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:58,026 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:58,026 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/C, priority=12, startTime=1733732157913; duration=0sec 2024-12-09T08:15:58,026 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:58,026 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:C 2024-12-09T08:15:58,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209aae44b309d8948f9b806c6002b99ecf6_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732157029/Put/seqid=0 2024-12-09T08:15:58,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742215_1391 (size=12304) 2024-12-09T08:15:58,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:58,063 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209aae44b309d8948f9b806c6002b99ecf6_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209aae44b309d8948f9b806c6002b99ecf6_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:58,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/c254bb8428ca41969434a75e4a8b4c1a, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:58,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/c254bb8428ca41969434a75e4a8b4c1a is 175, key is test_row_0/A:col10/1733732157029/Put/seqid=0 2024-12-09T08:15:58,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742216_1392 (size=31105) 2024-12-09T08:15:58,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T08:15:58,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732218253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732158227, max timestamp=1733732158259Num uniques 859; 2024-12-09T08:15:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732218260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:58,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732158227, max timestamp=1733732158259Num uniques 859; 2024-12-09T08:15:58,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732158227, max timestamp=1733732158259Num uniques 859; 2024-12-09T08:15:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:15:58,386 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/5b8b76d3142d4b1c9e11dd3a2cab1884 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/5b8b76d3142d4b1c9e11dd3a2cab1884 2024-12-09T08:15:58,390 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5081394aff7face3f62aba376aebd75e/A of 5081394aff7face3f62aba376aebd75e into 5b8b76d3142d4b1c9e11dd3a2cab1884(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:58,390 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:58,390 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/A, priority=12, startTime=1733732157913; duration=0sec 2024-12-09T08:15:58,390 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:58,390 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:A 2024-12-09T08:15:58,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732218379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732218380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732218397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,476 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/c254bb8428ca41969434a75e4a8b4c1a 2024-12-09T08:15:58,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/fa6f8913b86c47128cdd76afb4857a61 is 50, key is test_row_0/B:col10/1733732157029/Put/seqid=0 2024-12-09T08:15:58,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742217_1393 (size=12151) 2024-12-09T08:15:58,497 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/fa6f8913b86c47128cdd76afb4857a61 2024-12-09T08:15:58,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/77b3ad5847744dd087b1e69a0567faa8 is 50, key is test_row_0/C:col10/1733732157029/Put/seqid=0 2024-12-09T08:15:58,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732218502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732218502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742218_1394 (size=12151) 2024-12-09T08:15:58,519 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/77b3ad5847744dd087b1e69a0567faa8 2024-12-09T08:15:58,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732218512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/c254bb8428ca41969434a75e4a8b4c1a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/c254bb8428ca41969434a75e4a8b4c1a 2024-12-09T08:15:58,532 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/c254bb8428ca41969434a75e4a8b4c1a, entries=150, sequenceid=205, filesize=30.4 K 2024-12-09T08:15:58,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/fa6f8913b86c47128cdd76afb4857a61 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/fa6f8913b86c47128cdd76afb4857a61 2024-12-09T08:15:58,541 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/fa6f8913b86c47128cdd76afb4857a61, entries=150, sequenceid=205, filesize=11.9 K 2024-12-09T08:15:58,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/77b3ad5847744dd087b1e69a0567faa8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/77b3ad5847744dd087b1e69a0567faa8 2024-12-09T08:15:58,550 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/77b3ad5847744dd087b1e69a0567faa8, entries=150, sequenceid=205, filesize=11.9 K 2024-12-09T08:15:58,551 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5081394aff7face3f62aba376aebd75e in 529ms, sequenceid=205, compaction requested=false 2024-12-09T08:15:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-09T08:15:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-09T08:15:58,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-09T08:15:58,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4490 sec 2024-12-09T08:15:58,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4530 sec 2024-12-09T08:15:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:58,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T08:15:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:15:58,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e6d7c35a637c4dc7b96115b79ae0f5fa_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732158377/Put/seqid=0 2024-12-09T08:15:58,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742219_1395 (size=14794) 2024-12-09T08:15:58,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732218755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732218757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732218759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732218881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732218881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732218881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:58,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:58,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:58,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:59,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46560 deadline: 1733732219082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:59,087 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in 5081394aff7face3f62aba376aebd75e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:59,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46562 deadline: 1733732219082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:59,097 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 5081394aff7face3f62aba376aebd75e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:15:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:59,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732219101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:59,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732219101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:59,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732219101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:59,161 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:59,165 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e6d7c35a637c4dc7b96115b79ae0f5fa_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e6d7c35a637c4dc7b96115b79ae0f5fa_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:59,165 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/2f0c8eafb97e4656bacafbccb93968e3, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:59,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/2f0c8eafb97e4656bacafbccb93968e3 is 175, key is test_row_0/A:col10/1733732158377/Put/seqid=0 2024-12-09T08:15:59,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742220_1396 (size=39749) 2024-12-09T08:15:59,187 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/2f0c8eafb97e4656bacafbccb93968e3 2024-12-09T08:15:59,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/3ef0774f8b784324a39c30dbf2a64ae4 is 50, key is test_row_0/B:col10/1733732158377/Put/seqid=0 2024-12-09T08:15:59,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742221_1397 (size=12151) 2024-12-09T08:15:59,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/3ef0774f8b784324a39c30dbf2a64ae4 2024-12-09T08:15:59,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/3daa20c9db404fa09b5b33a55d8591e5 is 50, key is test_row_0/C:col10/1733732158377/Put/seqid=0 2024-12-09T08:15:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T08:15:59,251 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:15:59,252 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:15:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-09T08:15:59,254 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:15:59,256 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:15:59,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:15:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T08:15:59,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742222_1398 (size=12151) 2024-12-09T08:15:59,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/3daa20c9db404fa09b5b33a55d8591e5 2024-12-09T08:15:59,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/2f0c8eafb97e4656bacafbccb93968e3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2f0c8eafb97e4656bacafbccb93968e3 2024-12-09T08:15:59,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2f0c8eafb97e4656bacafbccb93968e3, entries=200, sequenceid=243, filesize=38.8 K 2024-12-09T08:15:59,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/3ef0774f8b784324a39c30dbf2a64ae4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/3ef0774f8b784324a39c30dbf2a64ae4 2024-12-09T08:15:59,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/3ef0774f8b784324a39c30dbf2a64ae4, entries=150, sequenceid=243, filesize=11.9 K 2024-12-09T08:15:59,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/3daa20c9db404fa09b5b33a55d8591e5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/3daa20c9db404fa09b5b33a55d8591e5 2024-12-09T08:15:59,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/3daa20c9db404fa09b5b33a55d8591e5, entries=150, sequenceid=243, filesize=11.9 K 2024-12-09T08:15:59,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 5081394aff7face3f62aba376aebd75e in 577ms, sequenceid=243, compaction requested=true 2024-12-09T08:15:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:15:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:59,305 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:15:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:15:59,305 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:59,306 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:59,306 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/B is initiating minor compaction (all files) 2024-12-09T08:15:59,306 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/B in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:59,306 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/685fe56202754ad4a5e994e03db1c1c3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/fa6f8913b86c47128cdd76afb4857a61, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/3ef0774f8b784324a39c30dbf2a64ae4] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=36.0 K 2024-12-09T08:15:59,306 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:59,306 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/A is initiating minor compaction (all files) 2024-12-09T08:15:59,307 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/A in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:59,307 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/5b8b76d3142d4b1c9e11dd3a2cab1884, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/c254bb8428ca41969434a75e4a8b4c1a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2f0c8eafb97e4656bacafbccb93968e3] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=100.0 K 2024-12-09T08:15:59,307 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:59,307 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/5b8b76d3142d4b1c9e11dd3a2cab1884, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/c254bb8428ca41969434a75e4a8b4c1a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2f0c8eafb97e4656bacafbccb93968e3] 2024-12-09T08:15:59,307 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 685fe56202754ad4a5e994e03db1c1c3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733732157007 2024-12-09T08:15:59,309 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b8b76d3142d4b1c9e11dd3a2cab1884, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733732157007 2024-12-09T08:15:59,309 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting fa6f8913b86c47128cdd76afb4857a61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733732157019 2024-12-09T08:15:59,309 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c254bb8428ca41969434a75e4a8b4c1a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733732157019 2024-12-09T08:15:59,311 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ef0774f8b784324a39c30dbf2a64ae4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733732158377 2024-12-09T08:15:59,311 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f0c8eafb97e4656bacafbccb93968e3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733732158377 2024-12-09T08:15:59,339 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#B#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:59,339 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/d0f3445ccaf64741891a0a1b3a062f47 is 50, key is test_row_0/B:col10/1733732158377/Put/seqid=0 2024-12-09T08:15:59,345 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:59,353 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120934fdb6e3852940e393396131ae792697_5081394aff7face3f62aba376aebd75e store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:59,354 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120934fdb6e3852940e393396131ae792697_5081394aff7face3f62aba376aebd75e, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:59,354 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120934fdb6e3852940e393396131ae792697_5081394aff7face3f62aba376aebd75e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:59,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742223_1399 (size=12697) 2024-12-09T08:15:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T08:15:59,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742224_1400 (size=4469) 2024-12-09T08:15:59,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T08:15:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:59,412 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:15:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:15:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:15:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:15:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:15:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:59,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:15:59,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209816026d45d354105af5814bc7ad6eba2_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732158753/Put/seqid=0 2024-12-09T08:15:59,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742225_1401 (size=12304) 2024-12-09T08:15:59,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732159478, max timestamp=1733732159497Num uniques 859; 2024-12-09T08:15:59,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:59,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732159478, max timestamp=1733732159497Num uniques 859; 2024-12-09T08:15:59,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:59,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732159478, max timestamp=1733732159497Num uniques 859; 2024-12-09T08:15:59,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:15:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T08:15:59,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733732159538, max timestamp=1733732159589Num uniques 1018; 2024-12-09T08:15:59,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T08:15:59,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733732159538, max timestamp=1733732159589Num uniques 1018; 2024-12-09T08:15:59,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733732159538, max timestamp=1733732159589Num uniques 1018; 2024-12-09T08:15:59,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T08:15:59,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T08:15:59,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732219674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:59,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:59,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732219674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:59,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732219681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:59,760 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/d0f3445ccaf64741891a0a1b3a062f47 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d0f3445ccaf64741891a0a1b3a062f47 2024-12-09T08:15:59,765 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#A#compaction#339 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:59,765 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/de3dafcd00a54bcd8601326f06e0b16e is 175, key is test_row_0/A:col10/1733732158377/Put/seqid=0 2024-12-09T08:15:59,766 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/B of 5081394aff7face3f62aba376aebd75e into d0f3445ccaf64741891a0a1b3a062f47(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:59,766 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:59,766 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/B, priority=13, startTime=1733732159305; duration=0sec 2024-12-09T08:15:59,766 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:15:59,766 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:B 2024-12-09T08:15:59,766 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:15:59,768 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:15:59,768 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/C is initiating minor compaction (all files) 2024-12-09T08:15:59,768 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/C in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:15:59,768 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/30510b30a730457f8c31cbab370289cb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/77b3ad5847744dd087b1e69a0567faa8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/3daa20c9db404fa09b5b33a55d8591e5] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=36.0 K 2024-12-09T08:15:59,769 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 30510b30a730457f8c31cbab370289cb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733732157007 2024-12-09T08:15:59,769 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 77b3ad5847744dd087b1e69a0567faa8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733732157019 2024-12-09T08:15:59,769 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3daa20c9db404fa09b5b33a55d8591e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733732158377 2024-12-09T08:15:59,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742226_1402 (size=31651) 2024-12-09T08:15:59,776 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/de3dafcd00a54bcd8601326f06e0b16e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/de3dafcd00a54bcd8601326f06e0b16e 2024-12-09T08:15:59,779 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#C#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:15:59,779 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/189d3a735c534e6ab56f24614c981b97 is 50, key is test_row_0/C:col10/1733732158377/Put/seqid=0 2024-12-09T08:15:59,782 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/A of 5081394aff7face3f62aba376aebd75e into de3dafcd00a54bcd8601326f06e0b16e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:59,782 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:59,782 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/A, priority=13, startTime=1733732159305; duration=0sec 2024-12-09T08:15:59,782 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:59,782 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:A 2024-12-09T08:15:59,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742227_1403 (size=12697) 2024-12-09T08:15:59,798 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/189d3a735c534e6ab56f24614c981b97 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/189d3a735c534e6ab56f24614c981b97 2024-12-09T08:15:59,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732219791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:59,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:59,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732219791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:59,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:59,805 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/C of 5081394aff7face3f62aba376aebd75e into 189d3a735c534e6ab56f24614c981b97(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:15:59,805 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:15:59,806 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/C, priority=13, startTime=1733732159305; duration=0sec 2024-12-09T08:15:59,806 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:15:59,806 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:C 2024-12-09T08:15:59,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:15:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732219801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:15:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:15:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:15:59,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:15:59,878 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209816026d45d354105af5814bc7ad6eba2_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209816026d45d354105af5814bc7ad6eba2_5081394aff7face3f62aba376aebd75e 2024-12-09T08:15:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/ef3a788702614fab9fc0031b5b0cc985, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:15:59,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/ef3a788702614fab9fc0031b5b0cc985 is 175, key is test_row_0/A:col10/1733732158753/Put/seqid=0 2024-12-09T08:15:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T08:15:59,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742228_1404 (size=31105) 2024-12-09T08:15:59,889 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/ef3a788702614fab9fc0031b5b0cc985 2024-12-09T08:15:59,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/622c18b53b524318bccfb798bae4cd47 is 50, key is test_row_0/B:col10/1733732158753/Put/seqid=0 2024-12-09T08:15:59,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742229_1405 (size=12151) 2024-12-09T08:15:59,921 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/622c18b53b524318bccfb798bae4cd47 2024-12-09T08:15:59,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/1f1943c1c17a4335a47bd14403c7fc9f is 50, key is test_row_0/C:col10/1733732158753/Put/seqid=0 2024-12-09T08:15:59,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742230_1406 (size=12151) 2024-12-09T08:16:00,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732220001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:00,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732220011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:00,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732220021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:00,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:00,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732220321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732220331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732220331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:00,347 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/1f1943c1c17a4335a47bd14403c7fc9f 2024-12-09T08:16:00,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/ef3a788702614fab9fc0031b5b0cc985 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef3a788702614fab9fc0031b5b0cc985 2024-12-09T08:16:00,353 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef3a788702614fab9fc0031b5b0cc985, entries=150, sequenceid=253, filesize=30.4 K 2024-12-09T08:16:00,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/622c18b53b524318bccfb798bae4cd47 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/622c18b53b524318bccfb798bae4cd47 2024-12-09T08:16:00,356 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/622c18b53b524318bccfb798bae4cd47, entries=150, sequenceid=253, filesize=11.9 K 2024-12-09T08:16:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/1f1943c1c17a4335a47bd14403c7fc9f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/1f1943c1c17a4335a47bd14403c7fc9f 2024-12-09T08:16:00,360 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/1f1943c1c17a4335a47bd14403c7fc9f, entries=150, sequenceid=253, filesize=11.9 K 2024-12-09T08:16:00,361 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5081394aff7face3f62aba376aebd75e in 948ms, sequenceid=253, compaction requested=false 2024-12-09T08:16:00,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:16:00,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:00,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-09T08:16:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-09T08:16:00,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-09T08:16:00,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1060 sec 2024-12-09T08:16:00,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.1120 sec 2024-12-09T08:16:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T08:16:00,391 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:00,391 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-09T08:16:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T08:16:00,393 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:00,393 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:00,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T08:16:00,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T08:16:00,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:00,545 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=134.18 KB heapSize=379.45 KB 2024-12-09T08:16:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:16:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:16:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:16:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:00,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209dac56fa6a0ce47528e3b7036cdc44e60_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732159672/Put/seqid=0 2024-12-09T08:16:00,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742231_1407 (size=12454) 2024-12-09T08:16:00,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:00,566 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209dac56fa6a0ce47528e3b7036cdc44e60_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209dac56fa6a0ce47528e3b7036cdc44e60_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/e71eb43844264c749a2f7493f9deba22, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/e71eb43844264c749a2f7493f9deba22 is 175, key is test_row_0/A:col10/1733732159672/Put/seqid=0 2024-12-09T08:16:00,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742232_1408 (size=31255) 2024-12-09T08:16:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T08:16:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:00,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:16:00,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732220857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732220867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:00,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732220869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:00,973 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/e71eb43844264c749a2f7493f9deba22 2024-12-09T08:16:00,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732220971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:00,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/1ff3c9aa99bf4670b67f52317264bfe5 is 50, key is test_row_0/B:col10/1733732159672/Put/seqid=0 2024-12-09T08:16:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:00,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732220981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:00,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732220981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:00,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742233_1409 (size=12301) 2024-12-09T08:16:01,003 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/1ff3c9aa99bf4670b67f52317264bfe5 2024-12-09T08:16:01,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/5799009e985b48f9a9dc3900b759f58c is 50, key is test_row_0/C:col10/1733732159672/Put/seqid=0 2024-12-09T08:16:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T08:16:01,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742234_1410 (size=12301) 2024-12-09T08:16:01,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732221193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732221201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732221201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,428 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/5799009e985b48f9a9dc3900b759f58c 2024-12-09T08:16:01,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/e71eb43844264c749a2f7493f9deba22 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/e71eb43844264c749a2f7493f9deba22 2024-12-09T08:16:01,435 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/e71eb43844264c749a2f7493f9deba22, entries=150, sequenceid=309, filesize=30.5 K 2024-12-09T08:16:01,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/1ff3c9aa99bf4670b67f52317264bfe5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/1ff3c9aa99bf4670b67f52317264bfe5 2024-12-09T08:16:01,439 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/1ff3c9aa99bf4670b67f52317264bfe5, entries=150, sequenceid=309, filesize=12.0 K 2024-12-09T08:16:01,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/5799009e985b48f9a9dc3900b759f58c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5799009e985b48f9a9dc3900b759f58c 2024-12-09T08:16:01,442 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5799009e985b48f9a9dc3900b759f58c, entries=150, sequenceid=309, filesize=12.0 K 2024-12-09T08:16:01,443 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~379.41 KB/388512, currentSize=46.96 KB/48090 for 5081394aff7face3f62aba376aebd75e in 898ms, sequenceid=309, compaction requested=true 2024-12-09T08:16:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:16:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-09T08:16:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-09T08:16:01,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-09T08:16:01,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0510 sec 2024-12-09T08:16:01,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.0550 sec 2024-12-09T08:16:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:01,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:16:01,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:16:01,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:01,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:16:01,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:01,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:16:01,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:01,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209576bc5133e6245d19567093ab302dcd0_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732161502/Put/seqid=0 2024-12-09T08:16:01,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742235_1411 (size=14994) 2024-12-09T08:16:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T08:16:01,531 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:01,532 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-09T08:16:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T08:16:01,533 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:01,533 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:01,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:01,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733732161532, max timestamp=1733732161555Num uniques 859; 2024-12-09T08:16:01,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T08:16:01,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733732161532, max timestamp=1733732161555Num uniques 859; 2024-12-09T08:16:01,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733732161532, max timestamp=1733732161555Num uniques 859; 2024-12-09T08:16:01,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T08:16:01,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T08:16:01,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732221598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732221599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:01,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732221600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T08:16:01,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T08:16:01,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:01,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:16:01,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:01,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:01,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:01,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732221710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:01,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732221711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:01,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:01,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732221711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:01,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T08:16:01,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:01,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:16:01,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:01,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:01,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T08:16:01,915 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:01,918 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209576bc5133e6245d19567093ab302dcd0_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209576bc5133e6245d19567093ab302dcd0_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:01,919 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/ef9aa5c0b1aa45919bd8eeb788f39db8, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:01,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/ef9aa5c0b1aa45919bd8eeb788f39db8 is 175, key is test_row_0/A:col10/1733732161502/Put/seqid=0 2024-12-09T08:16:01,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742236_1412 (size=39949) 2024-12-09T08:16:01,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732221921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:01,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732221921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732221921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:01,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T08:16:01,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:01,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:16:01,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:01,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:01,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T08:16:02,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:16:02,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T08:16:02,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732222231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:02,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732222241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:02,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732222241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:02,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T08:16:02,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:16:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,324 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/ef9aa5c0b1aa45919bd8eeb788f39db8 2024-12-09T08:16:02,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/c297567b38574c4cb82bd82daab16ee6 is 50, key is test_row_0/B:col10/1733732161502/Put/seqid=0 2024-12-09T08:16:02,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742237_1413 (size=12301) 2024-12-09T08:16:02,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/c297567b38574c4cb82bd82daab16ee6 2024-12-09T08:16:02,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/7c28da5474a74dc890aa98f2aa42998f is 50, key is test_row_0/C:col10/1733732161502/Put/seqid=0 2024-12-09T08:16:02,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742238_1414 (size=12301) 2024-12-09T08:16:02,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T08:16:02,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:16:02,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,511 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T08:16:02,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T08:16:02,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:16:02,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T08:16:02,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732222740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:02,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T08:16:02,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:16:02,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:02,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/7c28da5474a74dc890aa98f2aa42998f 2024-12-09T08:16:02,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/ef9aa5c0b1aa45919bd8eeb788f39db8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef9aa5c0b1aa45919bd8eeb788f39db8 2024-12-09T08:16:02,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef9aa5c0b1aa45919bd8eeb788f39db8, entries=200, sequenceid=320, filesize=39.0 K 2024-12-09T08:16:02,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/c297567b38574c4cb82bd82daab16ee6 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/c297567b38574c4cb82bd82daab16ee6 2024-12-09T08:16:02,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732222761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:02,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732222761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:02,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/c297567b38574c4cb82bd82daab16ee6, entries=150, sequenceid=320, filesize=12.0 K 2024-12-09T08:16:02,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/7c28da5474a74dc890aa98f2aa42998f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/7c28da5474a74dc890aa98f2aa42998f 2024-12-09T08:16:02,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/7c28da5474a74dc890aa98f2aa42998f, entries=150, sequenceid=320, filesize=12.0 K 2024-12-09T08:16:02,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5081394aff7face3f62aba376aebd75e in 1276ms, sequenceid=320, compaction requested=true 2024-12-09T08:16:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:16:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:02,780 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:16:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:02,780 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:16:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:02,781 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133960 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:16:02,781 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/A is initiating minor compaction (all files) 2024-12-09T08:16:02,781 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/A in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,781 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/de3dafcd00a54bcd8601326f06e0b16e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef3a788702614fab9fc0031b5b0cc985, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/e71eb43844264c749a2f7493f9deba22, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef9aa5c0b1aa45919bd8eeb788f39db8] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=130.8 K 2024-12-09T08:16:02,781 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,781 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/de3dafcd00a54bcd8601326f06e0b16e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef3a788702614fab9fc0031b5b0cc985, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/e71eb43844264c749a2f7493f9deba22, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef9aa5c0b1aa45919bd8eeb788f39db8] 2024-12-09T08:16:02,782 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:16:02,782 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/B is initiating minor compaction (all files) 2024-12-09T08:16:02,782 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/B in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,782 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d0f3445ccaf64741891a0a1b3a062f47, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/622c18b53b524318bccfb798bae4cd47, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/1ff3c9aa99bf4670b67f52317264bfe5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/c297567b38574c4cb82bd82daab16ee6] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=48.3 K 2024-12-09T08:16:02,783 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting de3dafcd00a54bcd8601326f06e0b16e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733732158377 2024-12-09T08:16:02,783 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef3a788702614fab9fc0031b5b0cc985, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733732158753 2024-12-09T08:16:02,783 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e71eb43844264c749a2f7493f9deba22, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733732159661 2024-12-09T08:16:02,784 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting d0f3445ccaf64741891a0a1b3a062f47, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733732158377 2024-12-09T08:16:02,784 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef9aa5c0b1aa45919bd8eeb788f39db8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733732160849 2024-12-09T08:16:02,784 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 622c18b53b524318bccfb798bae4cd47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733732158753 2024-12-09T08:16:02,785 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ff3c9aa99bf4670b67f52317264bfe5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733732159661 2024-12-09T08:16:02,785 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting c297567b38574c4cb82bd82daab16ee6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733732160849 2024-12-09T08:16:02,797 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:02,806 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#B#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:02,807 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/8f27fec4ddad4df0a0baa73a9040469c is 50, key is test_row_0/B:col10/1733732161502/Put/seqid=0 2024-12-09T08:16:02,808 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209e9256ac792104dcf9b436419df0e8017_5081394aff7face3f62aba376aebd75e store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:02,811 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209e9256ac792104dcf9b436419df0e8017_5081394aff7face3f62aba376aebd75e, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:02,811 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e9256ac792104dcf9b436419df0e8017_5081394aff7face3f62aba376aebd75e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:02,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742239_1415 (size=12983) 2024-12-09T08:16:02,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742240_1416 (size=4469) 2024-12-09T08:16:02,847 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#A#compaction#350 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:02,848 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/759d0a98660b49c2aca5c7874f2b8f6e is 175, key is test_row_0/A:col10/1733732161502/Put/seqid=0 2024-12-09T08:16:02,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742241_1417 (size=31937) 2024-12-09T08:16:02,861 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/759d0a98660b49c2aca5c7874f2b8f6e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/759d0a98660b49c2aca5c7874f2b8f6e 2024-12-09T08:16:02,865 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5081394aff7face3f62aba376aebd75e/A of 5081394aff7face3f62aba376aebd75e into 759d0a98660b49c2aca5c7874f2b8f6e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:02,865 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:16:02,865 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/A, priority=12, startTime=1733732162780; duration=0sec 2024-12-09T08:16:02,865 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:02,865 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:A 2024-12-09T08:16:02,865 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:16:02,866 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:16:02,866 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/C is initiating minor compaction (all files) 2024-12-09T08:16:02,866 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/C in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,866 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/189d3a735c534e6ab56f24614c981b97, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/1f1943c1c17a4335a47bd14403c7fc9f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5799009e985b48f9a9dc3900b759f58c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/7c28da5474a74dc890aa98f2aa42998f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=48.3 K 2024-12-09T08:16:02,866 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 189d3a735c534e6ab56f24614c981b97, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733732158377 2024-12-09T08:16:02,867 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f1943c1c17a4335a47bd14403c7fc9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733732158753 2024-12-09T08:16:02,867 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5799009e985b48f9a9dc3900b759f58c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733732159661 2024-12-09T08:16:02,867 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c28da5474a74dc890aa98f2aa42998f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733732160849 2024-12-09T08:16:02,875 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#C#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:02,875 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/3c093d194c4a4d9fb7ca062611d7e03f is 50, key is test_row_0/C:col10/1733732161502/Put/seqid=0 2024-12-09T08:16:02,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742242_1418 (size=12983) 2024-12-09T08:16:02,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T08:16:02,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:02,907 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-09T08:16:02,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:16:02,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:02,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:16:02,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:02,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:16:02,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:02,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096e63f3a68ddf46aa938fa0b34c131576_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732161599/Put/seqid=0 2024-12-09T08:16:02,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742243_1419 (size=12454) 2024-12-09T08:16:03,237 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/8f27fec4ddad4df0a0baa73a9040469c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/8f27fec4ddad4df0a0baa73a9040469c 2024-12-09T08:16:03,243 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5081394aff7face3f62aba376aebd75e/B of 5081394aff7face3f62aba376aebd75e into 8f27fec4ddad4df0a0baa73a9040469c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:03,243 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:16:03,243 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/B, priority=12, startTime=1733732162780; duration=0sec 2024-12-09T08:16:03,243 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:03,243 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:B 2024-12-09T08:16:03,288 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/3c093d194c4a4d9fb7ca062611d7e03f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/3c093d194c4a4d9fb7ca062611d7e03f 2024-12-09T08:16:03,292 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5081394aff7face3f62aba376aebd75e/C of 5081394aff7face3f62aba376aebd75e into 3c093d194c4a4d9fb7ca062611d7e03f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:03,292 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:16:03,292 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/C, priority=12, startTime=1733732162780; duration=0sec 2024-12-09T08:16:03,292 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:03,292 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:C 2024-12-09T08:16:03,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:03,332 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096e63f3a68ddf46aa938fa0b34c131576_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096e63f3a68ddf46aa938fa0b34c131576_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/a9ccaf3366cb4703b42c8748d4e81b58, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:03,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/a9ccaf3366cb4703b42c8748d4e81b58 is 175, key is test_row_0/A:col10/1733732161599/Put/seqid=0 2024-12-09T08:16:03,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742244_1420 (size=31255) 2024-12-09T08:16:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T08:16:03,738 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/a9ccaf3366cb4703b42c8748d4e81b58 2024-12-09T08:16:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/d5125501d68b47e8ba08bbc795456578 is 50, key is test_row_0/B:col10/1733732161599/Put/seqid=0 2024-12-09T08:16:03,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742245_1421 (size=12301) 2024-12-09T08:16:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:03,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. as already flushing 2024-12-09T08:16:03,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732223791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:03,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732223796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:03,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732223796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:03,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732223901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:03,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:03,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:03,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:03,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732223901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:03,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732223903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:04,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732224121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:04,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732224121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:04,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732224121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:04,148 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/d5125501d68b47e8ba08bbc795456578 2024-12-09T08:16:04,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/5c4078ed389549a0b9d935045a4cb460 is 50, key is test_row_0/C:col10/1733732161599/Put/seqid=0 2024-12-09T08:16:04,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742246_1422 (size=12301) 2024-12-09T08:16:04,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46548 deadline: 1733732224431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:04,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46516 deadline: 1733732224432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:04,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46554 deadline: 1733732224432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:04,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:04,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:04,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:04,466 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-12-09T08:16:04,466 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:04,466 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,466 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:04,467 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-12-09T08:16:04,467 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:04,467 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,467 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:04,472 INFO [Thread-1536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1536. 2024-12-09T08:16:04,472 DEBUG [Thread-1536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:04,472 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,472 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:04,477 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-12-09T08:16:04,477 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:04,477 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,477 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:04,478 INFO [Thread-1534 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1534. 2024-12-09T08:16:04,478 DEBUG [Thread-1534 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:04,478 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,478 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:04,557 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/5c4078ed389549a0b9d935045a4cb460 2024-12-09T08:16:04,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/a9ccaf3366cb4703b42c8748d4e81b58 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/a9ccaf3366cb4703b42c8748d4e81b58 2024-12-09T08:16:04,564 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/a9ccaf3366cb4703b42c8748d4e81b58, entries=150, sequenceid=355, filesize=30.5 K 2024-12-09T08:16:04,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/d5125501d68b47e8ba08bbc795456578 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d5125501d68b47e8ba08bbc795456578 2024-12-09T08:16:04,568 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d5125501d68b47e8ba08bbc795456578, entries=150, sequenceid=355, filesize=12.0 K 2024-12-09T08:16:04,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/5c4078ed389549a0b9d935045a4cb460 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5c4078ed389549a0b9d935045a4cb460 2024-12-09T08:16:04,573 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5c4078ed389549a0b9d935045a4cb460, entries=150, sequenceid=355, filesize=12.0 K 2024-12-09T08:16:04,574 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for 5081394aff7face3f62aba376aebd75e in 1667ms, sequenceid=355, compaction requested=false 2024-12-09T08:16:04,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:16:04,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:04,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-09T08:16:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-09T08:16:04,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-09T08:16:04,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0420 sec 2024-12-09T08:16:04,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.0460 sec 2024-12-09T08:16:04,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:04,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T08:16:04,942 INFO [Thread-1529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1529. 2024-12-09T08:16:04,942 DEBUG [Thread-1529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:04,942 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,942 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:04,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:16:04,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:04,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:16:04,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:04,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:16:04,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:04,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ca2afb1aff474a508f92a2a264dbf32e_5081394aff7face3f62aba376aebd75e is 50, key is test_row_0/A:col10/1733732163790/Put/seqid=0 2024-12-09T08:16:04,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742247_1423 (size=12454) 2024-12-09T08:16:04,952 INFO [Thread-1531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1531. 2024-12-09T08:16:04,952 DEBUG [Thread-1531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:04,952 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,952 INFO [Thread-1523 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1523. 2024-12-09T08:16:04,952 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,952 DEBUG [Thread-1523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:04,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:04,952 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,952 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:04,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:05,352 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:05,355 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ca2afb1aff474a508f92a2a264dbf32e_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ca2afb1aff474a508f92a2a264dbf32e_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:05,356 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/45b32c56250c4bfeaf2c896aec6ec2ff, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:05,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/45b32c56250c4bfeaf2c896aec6ec2ff is 175, key is test_row_0/A:col10/1733732163790/Put/seqid=0 2024-12-09T08:16:05,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742248_1424 (size=31255) 2024-12-09T08:16:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T08:16:05,692 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:05,761 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/45b32c56250c4bfeaf2c896aec6ec2ff 2024-12-09T08:16:05,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/e85cc8906a6b4db795bcbfeb0a0bc1c7 is 50, key is test_row_0/B:col10/1733732163790/Put/seqid=0 2024-12-09T08:16:05,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742249_1425 (size=12301) 2024-12-09T08:16:06,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/e85cc8906a6b4db795bcbfeb0a0bc1c7 2024-12-09T08:16:06,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/88a37d79952d4cd5869be27ad03620f3 is 50, key is test_row_0/C:col10/1733732163790/Put/seqid=0 2024-12-09T08:16:06,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742250_1426 (size=12301) 2024-12-09T08:16:06,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/88a37d79952d4cd5869be27ad03620f3 2024-12-09T08:16:06,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/45b32c56250c4bfeaf2c896aec6ec2ff as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/45b32c56250c4bfeaf2c896aec6ec2ff 2024-12-09T08:16:06,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/45b32c56250c4bfeaf2c896aec6ec2ff, entries=150, sequenceid=370, filesize=30.5 K 2024-12-09T08:16:06,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/e85cc8906a6b4db795bcbfeb0a0bc1c7 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/e85cc8906a6b4db795bcbfeb0a0bc1c7 2024-12-09T08:16:06,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/e85cc8906a6b4db795bcbfeb0a0bc1c7, entries=150, sequenceid=370, filesize=12.0 K 2024-12-09T08:16:06,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/88a37d79952d4cd5869be27ad03620f3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/88a37d79952d4cd5869be27ad03620f3 2024-12-09T08:16:06,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/88a37d79952d4cd5869be27ad03620f3, entries=150, sequenceid=370, filesize=12.0 K 2024-12-09T08:16:06,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=13.42 KB/13740 for 5081394aff7face3f62aba376aebd75e in 1669ms, sequenceid=370, compaction requested=true 2024-12-09T08:16:06,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:16:06,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:06,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:06,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:06,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:06,612 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:06,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5081394aff7face3f62aba376aebd75e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:06,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:06,612 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:06,612 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:06,612 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:06,612 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/B is initiating minor compaction (all files) 2024-12-09T08:16:06,612 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/A is initiating minor compaction (all files) 2024-12-09T08:16:06,613 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/B in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:06,613 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/A in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:06,613 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/8f27fec4ddad4df0a0baa73a9040469c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d5125501d68b47e8ba08bbc795456578, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/e85cc8906a6b4db795bcbfeb0a0bc1c7] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=36.7 K 2024-12-09T08:16:06,613 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/759d0a98660b49c2aca5c7874f2b8f6e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/a9ccaf3366cb4703b42c8748d4e81b58, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/45b32c56250c4bfeaf2c896aec6ec2ff] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=92.2 K 2024-12-09T08:16:06,613 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:06,613 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/759d0a98660b49c2aca5c7874f2b8f6e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/a9ccaf3366cb4703b42c8748d4e81b58, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/45b32c56250c4bfeaf2c896aec6ec2ff] 2024-12-09T08:16:06,613 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f27fec4ddad4df0a0baa73a9040469c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733732160849 2024-12-09T08:16:06,613 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 759d0a98660b49c2aca5c7874f2b8f6e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733732160849 2024-12-09T08:16:06,613 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting d5125501d68b47e8ba08bbc795456578, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733732161597 2024-12-09T08:16:06,613 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9ccaf3366cb4703b42c8748d4e81b58, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733732161597 2024-12-09T08:16:06,614 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45b32c56250c4bfeaf2c896aec6ec2ff, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733732163790 2024-12-09T08:16:06,614 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting e85cc8906a6b4db795bcbfeb0a0bc1c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733732163790 2024-12-09T08:16:06,622 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#B#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:06,622 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/31d58bd3f4b845c8a29bf7a5bb096a4f is 50, key is test_row_0/B:col10/1733732163790/Put/seqid=0 2024-12-09T08:16:06,625 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:06,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742251_1427 (size=13085) 2024-12-09T08:16:06,627 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412097313040c40f24a7890c13f454085fcbe_5081394aff7face3f62aba376aebd75e store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:06,629 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/31d58bd3f4b845c8a29bf7a5bb096a4f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/31d58bd3f4b845c8a29bf7a5bb096a4f 2024-12-09T08:16:06,632 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/B of 5081394aff7face3f62aba376aebd75e into 31d58bd3f4b845c8a29bf7a5bb096a4f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:06,632 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:16:06,632 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/B, priority=13, startTime=1733732166612; duration=0sec 2024-12-09T08:16:06,632 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:06,632 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:B 2024-12-09T08:16:06,633 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:06,633 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:06,633 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 5081394aff7face3f62aba376aebd75e/C is initiating minor compaction (all files) 2024-12-09T08:16:06,634 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5081394aff7face3f62aba376aebd75e/C in TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:06,634 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/3c093d194c4a4d9fb7ca062611d7e03f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5c4078ed389549a0b9d935045a4cb460, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/88a37d79952d4cd5869be27ad03620f3] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp, totalSize=36.7 K 2024-12-09T08:16:06,634 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c093d194c4a4d9fb7ca062611d7e03f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733732160849 2024-12-09T08:16:06,634 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c4078ed389549a0b9d935045a4cb460, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733732161597 2024-12-09T08:16:06,634 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 88a37d79952d4cd5869be27ad03620f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733732163790 2024-12-09T08:16:06,647 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#C#compaction#361 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:06,647 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/baad2ccedd90447dbdb7ad53c99a4bb9 is 50, key is test_row_0/C:col10/1733732163790/Put/seqid=0 2024-12-09T08:16:06,653 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412097313040c40f24a7890c13f454085fcbe_5081394aff7face3f62aba376aebd75e, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:06,653 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097313040c40f24a7890c13f454085fcbe_5081394aff7face3f62aba376aebd75e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:06,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742253_1429 (size=4469) 2024-12-09T08:16:06,665 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5081394aff7face3f62aba376aebd75e#A#compaction#360 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:06,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742252_1428 (size=13085) 2024-12-09T08:16:06,666 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/5f70881fff3445b983af84e22f673c89 is 175, key is test_row_0/A:col10/1733732163790/Put/seqid=0 2024-12-09T08:16:06,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742254_1430 (size=32039) 2024-12-09T08:16:06,674 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/baad2ccedd90447dbdb7ad53c99a4bb9 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/baad2ccedd90447dbdb7ad53c99a4bb9 2024-12-09T08:16:06,679 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/5f70881fff3445b983af84e22f673c89 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/5f70881fff3445b983af84e22f673c89 2024-12-09T08:16:06,680 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/C of 5081394aff7face3f62aba376aebd75e into baad2ccedd90447dbdb7ad53c99a4bb9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:06,680 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:16:06,680 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/C, priority=13, startTime=1733732166612; duration=0sec 2024-12-09T08:16:06,680 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:06,680 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:C 2024-12-09T08:16:06,684 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5081394aff7face3f62aba376aebd75e/A of 5081394aff7face3f62aba376aebd75e into 5f70881fff3445b983af84e22f673c89(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:06,684 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5081394aff7face3f62aba376aebd75e: 2024-12-09T08:16:06,684 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., storeName=5081394aff7face3f62aba376aebd75e/A, priority=13, startTime=1733732166612; duration=0sec 2024-12-09T08:16:06,684 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:06,684 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5081394aff7face3f62aba376aebd75e:A 2024-12-09T08:16:07,067 WARN [regionserver/5bda946059d1:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-09T08:16:07,068 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/25b59846fa2540189980aebe42e6719e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2528c7d115a54a36b3ca03d7cc3dc6f8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/dc9d0a07bafb471f887b3a12bf4e8789, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/bb360bf0511848179f9b0b2a6dbf34df, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/02478f2ebabf4c459d710eb5e760a70a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/bf5369f397434a20a7b071e55b254972, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/162295660d9846d5b0c61d58811bf395, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/aea477cb13b74b53bc79af9843afd57e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/30ab2dd6281b42208821632746072ae4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/000e4ee4a80d4918bae11fe59fff2bdd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/595a4afba1d743e79dd57eca03e26420, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/64980ee2ea5e49378c9e012991c7b025, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/55c47d7aabd5471aaea8ab700bc49662, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/5b8b76d3142d4b1c9e11dd3a2cab1884, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/c254bb8428ca41969434a75e4a8b4c1a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2f0c8eafb97e4656bacafbccb93968e3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/de3dafcd00a54bcd8601326f06e0b16e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef3a788702614fab9fc0031b5b0cc985, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/e71eb43844264c749a2f7493f9deba22, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef9aa5c0b1aa45919bd8eeb788f39db8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/759d0a98660b49c2aca5c7874f2b8f6e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/a9ccaf3366cb4703b42c8748d4e81b58, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/45b32c56250c4bfeaf2c896aec6ec2ff] to archive 2024-12-09T08:16:07,070 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:16:07,071 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/25b59846fa2540189980aebe42e6719e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/25b59846fa2540189980aebe42e6719e 2024-12-09T08:16:07,073 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2528c7d115a54a36b3ca03d7cc3dc6f8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2528c7d115a54a36b3ca03d7cc3dc6f8 2024-12-09T08:16:07,074 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/dc9d0a07bafb471f887b3a12bf4e8789 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/dc9d0a07bafb471f887b3a12bf4e8789 2024-12-09T08:16:07,075 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/bb360bf0511848179f9b0b2a6dbf34df to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/bb360bf0511848179f9b0b2a6dbf34df 2024-12-09T08:16:07,077 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/02478f2ebabf4c459d710eb5e760a70a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/02478f2ebabf4c459d710eb5e760a70a 2024-12-09T08:16:07,079 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/bf5369f397434a20a7b071e55b254972 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/bf5369f397434a20a7b071e55b254972 2024-12-09T08:16:07,080 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/162295660d9846d5b0c61d58811bf395 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/162295660d9846d5b0c61d58811bf395 2024-12-09T08:16:07,081 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/aea477cb13b74b53bc79af9843afd57e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/aea477cb13b74b53bc79af9843afd57e 2024-12-09T08:16:07,082 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/30ab2dd6281b42208821632746072ae4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/30ab2dd6281b42208821632746072ae4 2024-12-09T08:16:07,083 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/000e4ee4a80d4918bae11fe59fff2bdd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/000e4ee4a80d4918bae11fe59fff2bdd 2024-12-09T08:16:07,084 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/595a4afba1d743e79dd57eca03e26420 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/595a4afba1d743e79dd57eca03e26420 2024-12-09T08:16:07,085 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/64980ee2ea5e49378c9e012991c7b025 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/64980ee2ea5e49378c9e012991c7b025 2024-12-09T08:16:07,086 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/55c47d7aabd5471aaea8ab700bc49662 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/55c47d7aabd5471aaea8ab700bc49662 2024-12-09T08:16:07,086 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/5b8b76d3142d4b1c9e11dd3a2cab1884 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/5b8b76d3142d4b1c9e11dd3a2cab1884 2024-12-09T08:16:07,087 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/c254bb8428ca41969434a75e4a8b4c1a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/c254bb8428ca41969434a75e4a8b4c1a 2024-12-09T08:16:07,088 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2f0c8eafb97e4656bacafbccb93968e3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/2f0c8eafb97e4656bacafbccb93968e3 2024-12-09T08:16:07,089 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/de3dafcd00a54bcd8601326f06e0b16e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/de3dafcd00a54bcd8601326f06e0b16e 2024-12-09T08:16:07,090 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef3a788702614fab9fc0031b5b0cc985 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef3a788702614fab9fc0031b5b0cc985 2024-12-09T08:16:07,091 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/e71eb43844264c749a2f7493f9deba22 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/e71eb43844264c749a2f7493f9deba22 2024-12-09T08:16:07,091 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef9aa5c0b1aa45919bd8eeb788f39db8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/ef9aa5c0b1aa45919bd8eeb788f39db8 2024-12-09T08:16:07,092 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/759d0a98660b49c2aca5c7874f2b8f6e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/759d0a98660b49c2aca5c7874f2b8f6e 2024-12-09T08:16:07,093 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/a9ccaf3366cb4703b42c8748d4e81b58 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/a9ccaf3366cb4703b42c8748d4e81b58 2024-12-09T08:16:07,094 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/45b32c56250c4bfeaf2c896aec6ec2ff to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/45b32c56250c4bfeaf2c896aec6ec2ff 2024-12-09T08:16:07,097 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/66f2216f65a0499b85eef988941feffa, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/befe3c9d588b4d0e8f8872b830034d55, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/301d7c7027814094aaece750125a9cde, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d548155122614ada85450d1be577ea12, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/7d8c6ee5429c46a4b0a02e82af4f781b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/96213dab89f94b58a6b514ef0feb982d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/b603c39c4fe6440da595cb4948b78ade, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/83e291cdf4ab41eeb0e21013fbc4de69, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/33c4098a2400441fb776a008465e6952, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/b9285353a6ca468e9b77150f124a90d5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/37dbe69015d249b0b843f2f288f9231b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/3136d8da636c43cd8fc1462395d04840, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/685fe56202754ad4a5e994e03db1c1c3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/dc1aac7136004ebead9effdb51dcb0c4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/fa6f8913b86c47128cdd76afb4857a61, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d0f3445ccaf64741891a0a1b3a062f47, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/3ef0774f8b784324a39c30dbf2a64ae4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/622c18b53b524318bccfb798bae4cd47, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/1ff3c9aa99bf4670b67f52317264bfe5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/8f27fec4ddad4df0a0baa73a9040469c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/c297567b38574c4cb82bd82daab16ee6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d5125501d68b47e8ba08bbc795456578, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/e85cc8906a6b4db795bcbfeb0a0bc1c7] to archive 2024-12-09T08:16:07,098 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:16:07,100 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/66f2216f65a0499b85eef988941feffa to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/66f2216f65a0499b85eef988941feffa 2024-12-09T08:16:07,101 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/befe3c9d588b4d0e8f8872b830034d55 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/befe3c9d588b4d0e8f8872b830034d55 2024-12-09T08:16:07,102 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/301d7c7027814094aaece750125a9cde to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/301d7c7027814094aaece750125a9cde 2024-12-09T08:16:07,103 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d548155122614ada85450d1be577ea12 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d548155122614ada85450d1be577ea12 2024-12-09T08:16:07,104 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/7d8c6ee5429c46a4b0a02e82af4f781b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/7d8c6ee5429c46a4b0a02e82af4f781b 2024-12-09T08:16:07,105 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/96213dab89f94b58a6b514ef0feb982d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/96213dab89f94b58a6b514ef0feb982d 2024-12-09T08:16:07,106 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/b603c39c4fe6440da595cb4948b78ade to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/b603c39c4fe6440da595cb4948b78ade 2024-12-09T08:16:07,107 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/83e291cdf4ab41eeb0e21013fbc4de69 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/83e291cdf4ab41eeb0e21013fbc4de69 2024-12-09T08:16:07,108 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/33c4098a2400441fb776a008465e6952 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/33c4098a2400441fb776a008465e6952 2024-12-09T08:16:07,109 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/b9285353a6ca468e9b77150f124a90d5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/b9285353a6ca468e9b77150f124a90d5 2024-12-09T08:16:07,109 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/37dbe69015d249b0b843f2f288f9231b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/37dbe69015d249b0b843f2f288f9231b 2024-12-09T08:16:07,110 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/3136d8da636c43cd8fc1462395d04840 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/3136d8da636c43cd8fc1462395d04840 2024-12-09T08:16:07,111 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/685fe56202754ad4a5e994e03db1c1c3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/685fe56202754ad4a5e994e03db1c1c3 2024-12-09T08:16:07,111 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/dc1aac7136004ebead9effdb51dcb0c4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/dc1aac7136004ebead9effdb51dcb0c4 2024-12-09T08:16:07,112 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/fa6f8913b86c47128cdd76afb4857a61 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/fa6f8913b86c47128cdd76afb4857a61 2024-12-09T08:16:07,113 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d0f3445ccaf64741891a0a1b3a062f47 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d0f3445ccaf64741891a0a1b3a062f47 2024-12-09T08:16:07,114 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/3ef0774f8b784324a39c30dbf2a64ae4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/3ef0774f8b784324a39c30dbf2a64ae4 2024-12-09T08:16:07,114 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/622c18b53b524318bccfb798bae4cd47 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/622c18b53b524318bccfb798bae4cd47 2024-12-09T08:16:07,115 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/1ff3c9aa99bf4670b67f52317264bfe5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/1ff3c9aa99bf4670b67f52317264bfe5 2024-12-09T08:16:07,116 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/8f27fec4ddad4df0a0baa73a9040469c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/8f27fec4ddad4df0a0baa73a9040469c 2024-12-09T08:16:07,117 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/c297567b38574c4cb82bd82daab16ee6 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/c297567b38574c4cb82bd82daab16ee6 2024-12-09T08:16:07,118 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d5125501d68b47e8ba08bbc795456578 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/d5125501d68b47e8ba08bbc795456578 2024-12-09T08:16:07,118 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/e85cc8906a6b4db795bcbfeb0a0bc1c7 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/e85cc8906a6b4db795bcbfeb0a0bc1c7 2024-12-09T08:16:07,121 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5e224bd71de844f2a8916e7746f27a88, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/4d48196784374e8e947f0b05d6b63bc9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/bb11c0ea99a94f94acda22c1d23f7a5a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/c800e20e6149435f9e1cc39f35cd0097, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/b6ef12ce06014edd89565d53993bf029, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/c735aa743e6f4b9cb74758f6c4d0f968, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/edb8f7dd32714564a0698ef120495700, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/12e93a5a430343568d6237ee3e4aeecd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/429f59ca0cde4a72b47a30e40836897e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/4a3c88a90eb340d7b1ec6033a9f078ac, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/47a8b9d8a4184b5bae006413355b96cf, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/2e26d47b2dd24170bafcecd76a2b43af, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/30510b30a730457f8c31cbab370289cb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/ca1874c891314b1fb65aad002e2642db, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/77b3ad5847744dd087b1e69a0567faa8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/189d3a735c534e6ab56f24614c981b97, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/3daa20c9db404fa09b5b33a55d8591e5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/1f1943c1c17a4335a47bd14403c7fc9f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5799009e985b48f9a9dc3900b759f58c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/3c093d194c4a4d9fb7ca062611d7e03f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/7c28da5474a74dc890aa98f2aa42998f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5c4078ed389549a0b9d935045a4cb460, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/88a37d79952d4cd5869be27ad03620f3] to archive 2024-12-09T08:16:07,122 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:16:07,123 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5e224bd71de844f2a8916e7746f27a88 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5e224bd71de844f2a8916e7746f27a88 2024-12-09T08:16:07,124 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/4d48196784374e8e947f0b05d6b63bc9 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/4d48196784374e8e947f0b05d6b63bc9 2024-12-09T08:16:07,125 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/bb11c0ea99a94f94acda22c1d23f7a5a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/bb11c0ea99a94f94acda22c1d23f7a5a 2024-12-09T08:16:07,126 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/c800e20e6149435f9e1cc39f35cd0097 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/c800e20e6149435f9e1cc39f35cd0097 2024-12-09T08:16:07,127 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/b6ef12ce06014edd89565d53993bf029 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/b6ef12ce06014edd89565d53993bf029 2024-12-09T08:16:07,129 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/c735aa743e6f4b9cb74758f6c4d0f968 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/c735aa743e6f4b9cb74758f6c4d0f968 2024-12-09T08:16:07,130 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/edb8f7dd32714564a0698ef120495700 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/edb8f7dd32714564a0698ef120495700 2024-12-09T08:16:07,131 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/12e93a5a430343568d6237ee3e4aeecd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/12e93a5a430343568d6237ee3e4aeecd 2024-12-09T08:16:07,132 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/429f59ca0cde4a72b47a30e40836897e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/429f59ca0cde4a72b47a30e40836897e 2024-12-09T08:16:07,133 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/4a3c88a90eb340d7b1ec6033a9f078ac to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/4a3c88a90eb340d7b1ec6033a9f078ac 2024-12-09T08:16:07,134 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/47a8b9d8a4184b5bae006413355b96cf to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/47a8b9d8a4184b5bae006413355b96cf 2024-12-09T08:16:07,135 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/2e26d47b2dd24170bafcecd76a2b43af to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/2e26d47b2dd24170bafcecd76a2b43af 2024-12-09T08:16:07,136 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/30510b30a730457f8c31cbab370289cb to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/30510b30a730457f8c31cbab370289cb 2024-12-09T08:16:07,137 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/ca1874c891314b1fb65aad002e2642db to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/ca1874c891314b1fb65aad002e2642db 2024-12-09T08:16:07,138 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/77b3ad5847744dd087b1e69a0567faa8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/77b3ad5847744dd087b1e69a0567faa8 2024-12-09T08:16:07,138 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/189d3a735c534e6ab56f24614c981b97 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/189d3a735c534e6ab56f24614c981b97 2024-12-09T08:16:07,139 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/3daa20c9db404fa09b5b33a55d8591e5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/3daa20c9db404fa09b5b33a55d8591e5 2024-12-09T08:16:07,139 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/1f1943c1c17a4335a47bd14403c7fc9f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/1f1943c1c17a4335a47bd14403c7fc9f 2024-12-09T08:16:07,140 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5799009e985b48f9a9dc3900b759f58c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5799009e985b48f9a9dc3900b759f58c 2024-12-09T08:16:07,140 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/3c093d194c4a4d9fb7ca062611d7e03f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/3c093d194c4a4d9fb7ca062611d7e03f 2024-12-09T08:16:07,141 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/7c28da5474a74dc890aa98f2aa42998f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/7c28da5474a74dc890aa98f2aa42998f 2024-12-09T08:16:07,142 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5c4078ed389549a0b9d935045a4cb460 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/5c4078ed389549a0b9d935045a4cb460 2024-12-09T08:16:07,142 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/88a37d79952d4cd5869be27ad03620f3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/88a37d79952d4cd5869be27ad03620f3 2024-12-09T08:16:09,153 INFO [Thread-1527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1527. 2024-12-09T08:16:09,153 DEBUG [Thread-1527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:09,153 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:09,153 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:09,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:09,163 INFO [Thread-1525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1525. 2024-12-09T08:16:09,163 DEBUG [Thread-1525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:09,163 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:09,163 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:09,163 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1518 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4548 rows 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1517 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4548 rows 2024-12-09T08:16:09,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1513 2024-12-09T08:16:09,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4535 rows 2024-12-09T08:16:09,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1520 2024-12-09T08:16:09,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4553 rows 2024-12-09T08:16:09,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1514 2024-12-09T08:16:09,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4533 rows 2024-12-09T08:16:09,164 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T08:16:09,164 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:16:09,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:09,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:09,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:09,165 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T08:16:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T08:16:09,168 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732169168"}]},"ts":"1733732169168"} 2024-12-09T08:16:09,170 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T08:16:09,170 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T08:16:09,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T08:16:09,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, UNASSIGN}] 2024-12-09T08:16:09,172 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, UNASSIGN 2024-12-09T08:16:09,172 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=5081394aff7face3f62aba376aebd75e, regionState=CLOSING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:16:09,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, UNASSIGN because future has completed 2024-12-09T08:16:09,174 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:16:09,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:16:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T08:16:09,326 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:09,326 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:16:09,326 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 5081394aff7face3f62aba376aebd75e, disabling compactions & flushes 2024-12-09T08:16:09,326 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:09,327 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:09,327 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. after waiting 0 ms 2024-12-09T08:16:09,327 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:09,327 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 5081394aff7face3f62aba376aebd75e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-09T08:16:09,327 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=A 2024-12-09T08:16:09,327 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:09,328 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=B 2024-12-09T08:16:09,328 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:09,328 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5081394aff7face3f62aba376aebd75e, store=C 2024-12-09T08:16:09,328 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:09,337 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d3fa35f0366c41a7bd28be6b0ddfd6ef_5081394aff7face3f62aba376aebd75e is 50, key is test_row_1/A:col10/1733732169161/Put/seqid=0 2024-12-09T08:16:09,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742255_1431 (size=9914) 2024-12-09T08:16:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T08:16:09,741 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:09,745 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d3fa35f0366c41a7bd28be6b0ddfd6ef_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d3fa35f0366c41a7bd28be6b0ddfd6ef_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:09,746 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/cefb73ec0a18404b9a12fe86f9f88ca2, store: [table=TestAcidGuarantees family=A region=5081394aff7face3f62aba376aebd75e] 2024-12-09T08:16:09,746 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/cefb73ec0a18404b9a12fe86f9f88ca2 is 175, key is test_row_1/A:col10/1733732169161/Put/seqid=0 2024-12-09T08:16:09,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742256_1432 (size=22561) 2024-12-09T08:16:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T08:16:10,151 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/cefb73ec0a18404b9a12fe86f9f88ca2 2024-12-09T08:16:10,162 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/caff2c4f492549a8a11de61b6cc410d5 is 50, key is test_row_1/B:col10/1733732169161/Put/seqid=0 2024-12-09T08:16:10,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742257_1433 (size=9857) 2024-12-09T08:16:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T08:16:10,567 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/caff2c4f492549a8a11de61b6cc410d5 2024-12-09T08:16:10,575 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/f050e3c2ca014523a83ed4df2d366065 is 50, key is test_row_1/C:col10/1733732169161/Put/seqid=0 2024-12-09T08:16:10,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742258_1434 (size=9857) 2024-12-09T08:16:10,983 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/f050e3c2ca014523a83ed4df2d366065 2024-12-09T08:16:10,988 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/A/cefb73ec0a18404b9a12fe86f9f88ca2 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/cefb73ec0a18404b9a12fe86f9f88ca2 2024-12-09T08:16:10,992 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/cefb73ec0a18404b9a12fe86f9f88ca2, entries=100, sequenceid=380, filesize=22.0 K 2024-12-09T08:16:10,993 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/B/caff2c4f492549a8a11de61b6cc410d5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/caff2c4f492549a8a11de61b6cc410d5 2024-12-09T08:16:10,997 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/caff2c4f492549a8a11de61b6cc410d5, entries=100, sequenceid=380, filesize=9.6 K 2024-12-09T08:16:10,997 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/.tmp/C/f050e3c2ca014523a83ed4df2d366065 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/f050e3c2ca014523a83ed4df2d366065 2024-12-09T08:16:11,001 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/f050e3c2ca014523a83ed4df2d366065, entries=100, sequenceid=380, filesize=9.6 K 2024-12-09T08:16:11,002 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5081394aff7face3f62aba376aebd75e in 1675ms, sequenceid=380, compaction requested=false 2024-12-09T08:16:11,005 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/recovered.edits/383.seqid, newMaxSeqId=383, maxSeqId=4 2024-12-09T08:16:11,006 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. 2024-12-09T08:16:11,006 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 5081394aff7face3f62aba376aebd75e: Waiting for close lock at 1733732169326Running coprocessor pre-close hooks at 1733732169326Disabling compacts and flushes for region at 1733732169326Disabling writes for close at 1733732169327 (+1 ms)Obtaining lock to block concurrent updates at 1733732169327Preparing flush snapshotting stores in 5081394aff7face3f62aba376aebd75e at 1733732169327Finished memstore snapshotting TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733732169328 (+1 ms)Flushing stores of TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e. at 1733732169329 (+1 ms)Flushing 5081394aff7face3f62aba376aebd75e/A: creating writer at 1733732169329Flushing mob file 5081394aff7face3f62aba376aebd75e/A: appending metadata at 1733732169336 (+7 ms)Flushing mob file 5081394aff7face3f62aba376aebd75e/A: closing flushed file at 1733732169336Flushing 5081394aff7face3f62aba376aebd75e/A: appending metadata at 1733732169746 (+410 ms)Flushing 5081394aff7face3f62aba376aebd75e/A: closing flushed file at 1733732169746Flushing 5081394aff7face3f62aba376aebd75e/B: creating writer at 1733732170159 (+413 ms)Flushing 5081394aff7face3f62aba376aebd75e/B: appending metadata at 1733732170162 (+3 ms)Flushing 5081394aff7face3f62aba376aebd75e/B: closing flushed file at 1733732170162Flushing 5081394aff7face3f62aba376aebd75e/C: creating writer at 1733732170571 (+409 ms)Flushing 5081394aff7face3f62aba376aebd75e/C: appending metadata at 1733732170574 (+3 ms)Flushing 5081394aff7face3f62aba376aebd75e/C: closing flushed file at 1733732170574Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c953af0: reopening flushed file at 1733732170987 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a904608: reopening flushed file at 1733732170992 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42649828: reopening flushed file at 1733732170997 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5081394aff7face3f62aba376aebd75e in 1675ms, sequenceid=380, compaction requested=false at 1733732171002 (+5 ms)Writing region close event to WAL at 1733732171003 (+1 ms)Running coprocessor post-close hooks at 1733732171006 (+3 ms)Closed at 1733732171006 2024-12-09T08:16:11,007 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,008 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=5081394aff7face3f62aba376aebd75e, regionState=CLOSED 2024-12-09T08:16:11,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:16:11,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-09T08:16:11,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 5081394aff7face3f62aba376aebd75e, server=5bda946059d1,43255,1733732044053 in 1.8350 sec 2024-12-09T08:16:11,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-12-09T08:16:11,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5081394aff7face3f62aba376aebd75e, UNASSIGN in 1.8400 sec 2024-12-09T08:16:11,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-09T08:16:11,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8420 sec 2024-12-09T08:16:11,015 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732171015"}]},"ts":"1733732171015"} 2024-12-09T08:16:11,016 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T08:16:11,016 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T08:16:11,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8510 sec 2024-12-09T08:16:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T08:16:11,312 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:11,313 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T08:16:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:11,318 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T08:16:11,320 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:11,325 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,328 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/recovered.edits] 2024-12-09T08:16:11,332 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/5f70881fff3445b983af84e22f673c89 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/5f70881fff3445b983af84e22f673c89 2024-12-09T08:16:11,334 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/cefb73ec0a18404b9a12fe86f9f88ca2 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/A/cefb73ec0a18404b9a12fe86f9f88ca2 2024-12-09T08:16:11,338 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/31d58bd3f4b845c8a29bf7a5bb096a4f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/31d58bd3f4b845c8a29bf7a5bb096a4f 2024-12-09T08:16:11,339 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/caff2c4f492549a8a11de61b6cc410d5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/B/caff2c4f492549a8a11de61b6cc410d5 2024-12-09T08:16:11,343 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/baad2ccedd90447dbdb7ad53c99a4bb9 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/baad2ccedd90447dbdb7ad53c99a4bb9 2024-12-09T08:16:11,344 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/f050e3c2ca014523a83ed4df2d366065 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/C/f050e3c2ca014523a83ed4df2d366065 2024-12-09T08:16:11,347 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/recovered.edits/383.seqid to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e/recovered.edits/383.seqid 2024-12-09T08:16:11,348 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,348 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T08:16:11,349 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T08:16:11,350 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T08:16:11,354 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120929785db219e540799e99c36fcfb358ea_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120929785db219e540799e99c36fcfb358ea_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,356 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092eaeb32e714a4dba90ff9a99eeeff06b_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092eaeb32e714a4dba90ff9a99eeeff06b_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,358 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093d7d27e00cc6417ebc0dc253b42aa71b_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093d7d27e00cc6417ebc0dc253b42aa71b_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,359 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094ee672b83c0b48468978608b850e3ac7_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094ee672b83c0b48468978608b850e3ac7_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,361 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209576bc5133e6245d19567093ab302dcd0_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209576bc5133e6245d19567093ab302dcd0_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,362 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096ba1090985a14226aa8fd90ff7a86cab_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096ba1090985a14226aa8fd90ff7a86cab_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,362 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096e63f3a68ddf46aa938fa0b34c131576_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096e63f3a68ddf46aa938fa0b34c131576_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,363 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097d59194c1bde4776b9c763e6faff77bc_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097d59194c1bde4776b9c763e6faff77bc_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,364 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209816026d45d354105af5814bc7ad6eba2_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209816026d45d354105af5814bc7ad6eba2_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,364 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120988997535cb974af3ae8143a382fabbbe_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120988997535cb974af3ae8143a382fabbbe_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,365 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209aae44b309d8948f9b806c6002b99ecf6_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209aae44b309d8948f9b806c6002b99ecf6_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,366 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c7a016b2672e493aa948bc8d4df58667_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c7a016b2672e493aa948bc8d4df58667_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,367 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ca2afb1aff474a508f92a2a264dbf32e_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ca2afb1aff474a508f92a2a264dbf32e_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,367 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d3fa35f0366c41a7bd28be6b0ddfd6ef_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d3fa35f0366c41a7bd28be6b0ddfd6ef_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,368 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209dac56fa6a0ce47528e3b7036cdc44e60_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209dac56fa6a0ce47528e3b7036cdc44e60_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,369 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e52dd219fec34ade8f2fe7a3261b1e33_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e52dd219fec34ade8f2fe7a3261b1e33_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,369 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e6d7c35a637c4dc7b96115b79ae0f5fa_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e6d7c35a637c4dc7b96115b79ae0f5fa_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,370 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f3b306ed5d2d4309952016a995aba728_5081394aff7face3f62aba376aebd75e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f3b306ed5d2d4309952016a995aba728_5081394aff7face3f62aba376aebd75e 2024-12-09T08:16:11,370 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T08:16:11,372 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:11,373 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T08:16:11,375 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T08:16:11,376 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:11,376 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T08:16:11,376 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733732171376"}]},"ts":"9223372036854775807"} 2024-12-09T08:16:11,377 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T08:16:11,378 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5081394aff7face3f62aba376aebd75e, NAME => 'TestAcidGuarantees,,1733732141334.5081394aff7face3f62aba376aebd75e.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T08:16:11,378 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T08:16:11,378 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733732171378"}]},"ts":"9223372036854775807"} 2024-12-09T08:16:11,379 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T08:16:11,380 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:11,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-12-09T08:16:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T08:16:11,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T08:16:11,432 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:11,441 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 250), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=487 (was 397) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4486 (was 4117) - AvailableMemoryMB LEAK? - 2024-12-09T08:16:11,451 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=487, ProcessCount=11, AvailableMemoryMB=4486 2024-12-09T08:16:11,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T08:16:11,453 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:16:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:11,455 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T08:16:11,455 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:11,455 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-12-09T08:16:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T08:16:11,456 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T08:16:11,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742259_1435 (size=1061) 2024-12-09T08:16:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T08:16:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T08:16:11,867 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447 2024-12-09T08:16:11,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742260_1436 (size=53) 2024-12-09T08:16:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T08:16:12,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:16:12,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6b37acb92571896382ef9b774ba15dad, disabling compactions & flushes 2024-12-09T08:16:12,275 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:12,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:12,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. after waiting 0 ms 2024-12-09T08:16:12,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:12,275 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:12,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6b37acb92571896382ef9b774ba15dad: Waiting for close lock at 1733732172275Disabling compacts and flushes for region at 1733732172275Disabling writes for close at 1733732172275Writing region close event to WAL at 1733732172275Closed at 1733732172275 2024-12-09T08:16:12,276 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T08:16:12,277 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733732172276"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733732172276"}]},"ts":"1733732172276"} 2024-12-09T08:16:12,278 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T08:16:12,278 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T08:16:12,278 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732172278"}]},"ts":"1733732172278"} 2024-12-09T08:16:12,280 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T08:16:12,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b37acb92571896382ef9b774ba15dad, ASSIGN}] 2024-12-09T08:16:12,281 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b37acb92571896382ef9b774ba15dad, ASSIGN 2024-12-09T08:16:12,281 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b37acb92571896382ef9b774ba15dad, ASSIGN; state=OFFLINE, location=5bda946059d1,43255,1733732044053; forceNewPlan=false, retain=false 2024-12-09T08:16:12,432 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=6b37acb92571896382ef9b774ba15dad, regionState=OPENING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:16:12,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b37acb92571896382ef9b774ba15dad, ASSIGN because future has completed 2024-12-09T08:16:12,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:16:12,590 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:12,590 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:16:12,590 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,590 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:16:12,590 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,590 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T08:16:12,591 INFO [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,592 INFO [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:16:12,592 INFO [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b37acb92571896382ef9b774ba15dad columnFamilyName A 2024-12-09T08:16:12,592 DEBUG [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:12,593 INFO [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] regionserver.HStore(327): Store=6b37acb92571896382ef9b774ba15dad/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:16:12,593 INFO [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,594 INFO [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:16:12,594 INFO [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b37acb92571896382ef9b774ba15dad columnFamilyName B 2024-12-09T08:16:12,594 DEBUG [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:12,594 INFO [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] regionserver.HStore(327): Store=6b37acb92571896382ef9b774ba15dad/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:16:12,594 INFO [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,595 INFO [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:16:12,595 INFO [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b37acb92571896382ef9b774ba15dad columnFamilyName C 2024-12-09T08:16:12,595 DEBUG [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:12,596 INFO [StoreOpener-6b37acb92571896382ef9b774ba15dad-1 {}] regionserver.HStore(327): Store=6b37acb92571896382ef9b774ba15dad/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:16:12,596 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:12,596 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,596 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,597 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,597 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,597 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,598 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:16:12,599 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,600 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:16:12,600 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 6b37acb92571896382ef9b774ba15dad; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69648951, jitterRate=0.03785024583339691}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:16:12,601 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:12,601 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 6b37acb92571896382ef9b774ba15dad: Running coprocessor pre-open hook at 1733732172590Writing region info on filesystem at 1733732172590Initializing all the Stores at 1733732172591 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732172591Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732172591Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732172591Cleaning up temporary data from old regions at 1733732172597 (+6 ms)Running coprocessor post-open hooks at 1733732172601 (+4 ms)Region opened successfully at 1733732172601 2024-12-09T08:16:12,602 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., pid=114, masterSystemTime=1733732172587 2024-12-09T08:16:12,604 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:12,604 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:12,604 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=6b37acb92571896382ef9b774ba15dad, regionState=OPEN, openSeqNum=2, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:16:12,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:16:12,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-09T08:16:12,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 in 172 msec 2024-12-09T08:16:12,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-09T08:16:12,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b37acb92571896382ef9b774ba15dad, ASSIGN in 328 msec 2024-12-09T08:16:12,611 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T08:16:12,611 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732172611"}]},"ts":"1733732172611"} 2024-12-09T08:16:12,612 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T08:16:12,614 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T08:16:12,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-12-09T08:16:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T08:16:13,601 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:13,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a364c5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:13,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:13,603 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1104780, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:13,604 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:13,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1562df7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,605 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:13,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:13,608 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T08:16:13,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 5bda946059d1,45611,1733732043366 2024-12-09T08:16:13,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2c3b47d6 2024-12-09T08:16:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T08:16:13,610 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T08:16:13,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6516064e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,611 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:13,612 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:13,612 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e0a702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:13,614 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3b62e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:13,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:13,616 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29707c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:13,617 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:13,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2838ddd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,618 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:13,618 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:13,619 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6382bcc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:13,620 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17cb1c3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:13,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:13,621 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43892d66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:13,622 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:13,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a01e41d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,622 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:13,622 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:13,623 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74eb7bf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:13,624 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:13,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@574f4cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:13,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:13,625 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:13,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:13,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:13,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1303cfad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:13,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:13,626 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@524b1ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:13,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:13,628 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f003170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:13,629 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25946ab5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:13,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:13,631 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d1ff919, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:13,633 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45825c3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:13,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:13,635 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3ca814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:13,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:13,636 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ed7aef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:13,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:13,638 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758440b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:13,639 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:13,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b585966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:13,640 DEBUG [Thread-1968 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:13,640 DEBUG [Thread-1966 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:13,640 DEBUG [Thread-1962 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:13,640 DEBUG [Thread-1975 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:13,640 DEBUG [Thread-1981 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:13,640 DEBUG [Thread-1964 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:13,641 DEBUG [Thread-1970 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:13,641 DEBUG [Thread-1977 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:13,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:13,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:13,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:13,642 DEBUG [Thread-1977 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:13,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:13,642 DEBUG [Thread-1979 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:13,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:13,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:13,643 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:13,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:13,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:13,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:16:13,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:16:13,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:16:13,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:16:13,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:16:13,645 DEBUG [Thread-1973 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:13,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:13,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:16:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:16:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-09T08:16:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:16:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:16:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T08:16:13,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:13,649 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:13,650 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2] 2024-12-09T08:16:13,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:13,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:16:13,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:13,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:13,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:13,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:13,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:13,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:13,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/a9df4b01bb2c4c45ba9777ea9e625e94 is 50, key is test_row_0/A:col10/1733732173650/Put/seqid=0 2024-12-09T08:16:13,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742261_1437 (size=12001) 2024-12-09T08:16:13,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733732173708, max timestamp=1733732173724Num uniques 859; 2024-12-09T08:16:13,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T08:16:13,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733732173708, max timestamp=1733732173724Num uniques 859; 2024-12-09T08:16:13,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T08:16:13,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733732173712, max timestamp=1733732173724Num uniques 859; 2024-12-09T08:16:13,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T08:16:13,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:13,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732233745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:13,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T08:16:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:13,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732233749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:13,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:13,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732233750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:13,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:13,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732233750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732233751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:13,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T08:16:13,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:13,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:13,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:13,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732233861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732233861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:13,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732233871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:13,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732233871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:13,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732233872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:13,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T08:16:13,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:13,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:13,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:13,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:13,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T08:16:14,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732234071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/a9df4b01bb2c4c45ba9777ea9e625e94 2024-12-09T08:16:14,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732234081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732234091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732234092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732234094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T08:16:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:14,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:14,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/2e2c7e53b65b444080402acd97dc62fb is 50, key is test_row_0/B:col10/1733732173650/Put/seqid=0 2024-12-09T08:16:14,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742262_1438 (size=12001) 2024-12-09T08:16:14,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T08:16:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:14,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T08:16:14,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732234381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732234401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732234401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732234401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T08:16:14,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:14,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:14,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:14,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:14,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732234412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/2e2c7e53b65b444080402acd97dc62fb 2024-12-09T08:16:14,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T08:16:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:14,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:14,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/a70b67aa4bc9479c976dd199374a134c is 50, key is test_row_0/C:col10/1733732173650/Put/seqid=0 2024-12-09T08:16:14,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742263_1439 (size=12001) 2024-12-09T08:16:14,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/a70b67aa4bc9479c976dd199374a134c 2024-12-09T08:16:14,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/a9df4b01bb2c4c45ba9777ea9e625e94 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/a9df4b01bb2c4c45ba9777ea9e625e94 2024-12-09T08:16:14,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/a9df4b01bb2c4c45ba9777ea9e625e94, entries=150, sequenceid=11, filesize=11.7 K 2024-12-09T08:16:14,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/2e2c7e53b65b444080402acd97dc62fb as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2e2c7e53b65b444080402acd97dc62fb 2024-12-09T08:16:14,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2e2c7e53b65b444080402acd97dc62fb, entries=150, sequenceid=11, filesize=11.7 K 2024-12-09T08:16:14,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/a70b67aa4bc9479c976dd199374a134c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/a70b67aa4bc9479c976dd199374a134c 2024-12-09T08:16:14,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/a70b67aa4bc9479c976dd199374a134c, entries=150, sequenceid=11, filesize=11.7 K 2024-12-09T08:16:14,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=142.73 KB/146155 for 6b37acb92571896382ef9b774ba15dad in 1009ms, sequenceid=11, compaction requested=false 2024-12-09T08:16:14,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:14,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T08:16:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:14,736 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=142.73 KB heapSize=404.38 KB 2024-12-09T08:16:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:14,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/4f86c2a83aed4879bcb79f0a3218c591 is 50, key is test_row_0/A:col10/1733732173748/Put/seqid=0 2024-12-09T08:16:14,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742264_1440 (size=12001) 2024-12-09T08:16:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T08:16:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:14,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:14,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732234921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732234921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732234924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732234931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:14,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732234934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732235041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732235041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732235051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,172 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/4f86c2a83aed4879bcb79f0a3218c591 2024-12-09T08:16:15,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/c9e63a69e7db4243b6dd9d503741c5c7 is 50, key is test_row_0/B:col10/1733732173748/Put/seqid=0 2024-12-09T08:16:15,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742265_1441 (size=12001) 2024-12-09T08:16:15,206 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/c9e63a69e7db4243b6dd9d503741c5c7 2024-12-09T08:16:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/965a00fc0f104efcae1d15c325186c77 is 50, key is test_row_0/C:col10/1733732173748/Put/seqid=0 2024-12-09T08:16:15,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742266_1442 (size=12001) 2024-12-09T08:16:15,253 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/965a00fc0f104efcae1d15c325186c77 2024-12-09T08:16:15,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732235251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/4f86c2a83aed4879bcb79f0a3218c591 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/4f86c2a83aed4879bcb79f0a3218c591 2024-12-09T08:16:15,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732235261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732235261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,270 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/4f86c2a83aed4879bcb79f0a3218c591, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T08:16:15,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/c9e63a69e7db4243b6dd9d503741c5c7 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/c9e63a69e7db4243b6dd9d503741c5c7 2024-12-09T08:16:15,275 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/c9e63a69e7db4243b6dd9d503741c5c7, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T08:16:15,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/965a00fc0f104efcae1d15c325186c77 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/965a00fc0f104efcae1d15c325186c77 2024-12-09T08:16:15,281 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/965a00fc0f104efcae1d15c325186c77, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T08:16:15,282 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~142.73 KB/146155, heapSize ~404.34 KB/414040, currentSize=40.25 KB/41220 for 6b37acb92571896382ef9b774ba15dad in 546ms, sequenceid=48, compaction requested=false 2024-12-09T08:16:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-09T08:16:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-09T08:16:15,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-09T08:16:15,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6330 sec 2024-12-09T08:16:15,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.6440 sec 2024-12-09T08:16:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:15,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:16:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:15,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/e4320f829596485e8a4d493f3390c3ea is 50, key is test_row_0/A:col10/1733732175574/Put/seqid=0 2024-12-09T08:16:15,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742267_1443 (size=14341) 2024-12-09T08:16:15,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732235642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732235643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732235645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732235751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732235761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732235761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T08:16:15,791 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:15,791 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-09T08:16:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T08:16:15,792 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:15,793 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:15,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T08:16:15,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T08:16:15,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:15,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:15,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:15,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:15,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:15,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732235951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732235951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,955 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 6b37acb92571896382ef9b774ba15dad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,955 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 6b37acb92571896382ef9b774ba15dad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732235961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732235970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732235971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:15,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/e4320f829596485e8a4d493f3390c3ea 2024-12-09T08:16:15,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/fb5b3254305f426798057942aa548198 is 50, key is test_row_0/B:col10/1733732175574/Put/seqid=0 2024-12-09T08:16:16,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742268_1444 (size=12001) 2024-12-09T08:16:16,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T08:16:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T08:16:16,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T08:16:16,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:16,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732236271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732236281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732236281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:16,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T08:16:16,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:16,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/fb5b3254305f426798057942aa548198 2024-12-09T08:16:16,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/bd6d0546bd0f4871b60bcadd62a05df3 is 50, key is test_row_0/C:col10/1733732175574/Put/seqid=0 2024-12-09T08:16:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T08:16:16,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742269_1445 (size=12001) 2024-12-09T08:16:16,525 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:16:16,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T08:16:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:16,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T08:16:16,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:16,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:16,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732236781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:16,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732236801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:16,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732236801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:16,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/bd6d0546bd0f4871b60bcadd62a05df3 2024-12-09T08:16:16,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/e4320f829596485e8a4d493f3390c3ea as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/e4320f829596485e8a4d493f3390c3ea 2024-12-09T08:16:16,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/e4320f829596485e8a4d493f3390c3ea, entries=200, sequenceid=60, filesize=14.0 K 2024-12-09T08:16:16,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/fb5b3254305f426798057942aa548198 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/fb5b3254305f426798057942aa548198 2024-12-09T08:16:16,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/fb5b3254305f426798057942aa548198, entries=150, sequenceid=60, filesize=11.7 K 2024-12-09T08:16:16,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/bd6d0546bd0f4871b60bcadd62a05df3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/bd6d0546bd0f4871b60bcadd62a05df3 2024-12-09T08:16:16,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/bd6d0546bd0f4871b60bcadd62a05df3, entries=150, sequenceid=60, filesize=11.7 K 2024-12-09T08:16:16,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6b37acb92571896382ef9b774ba15dad in 1265ms, sequenceid=60, compaction requested=true 2024-12-09T08:16:16,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:16,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:16,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:16,841 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:16,841 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:16,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:16,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:16,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:16,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:16,842 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:16,842 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/A is initiating minor compaction (all files) 2024-12-09T08:16:16,842 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/A in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,842 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/a9df4b01bb2c4c45ba9777ea9e625e94, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/4f86c2a83aed4879bcb79f0a3218c591, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/e4320f829596485e8a4d493f3390c3ea] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=37.4 K 2024-12-09T08:16:16,843 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:16,843 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/B is initiating minor compaction (all files) 2024-12-09T08:16:16,843 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/B in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,843 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9df4b01bb2c4c45ba9777ea9e625e94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733732173645 2024-12-09T08:16:16,843 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2e2c7e53b65b444080402acd97dc62fb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/c9e63a69e7db4243b6dd9d503741c5c7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/fb5b3254305f426798057942aa548198] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=35.2 K 2024-12-09T08:16:16,843 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f86c2a83aed4879bcb79f0a3218c591, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733732173748 2024-12-09T08:16:16,843 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e2c7e53b65b444080402acd97dc62fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733732173645 2024-12-09T08:16:16,843 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4320f829596485e8a4d493f3390c3ea, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733732174907 2024-12-09T08:16:16,843 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting c9e63a69e7db4243b6dd9d503741c5c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733732173748 2024-12-09T08:16:16,843 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting fb5b3254305f426798057942aa548198, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733732175574 2024-12-09T08:16:16,848 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#B#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:16,848 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/24bd5dc35dd74e21a2de0cff6ef3acf6 is 50, key is test_row_0/B:col10/1733732175574/Put/seqid=0 2024-12-09T08:16:16,849 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:16,849 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/7ea2e25e868c4578a5f5d3f700a4969e is 50, key is test_row_0/A:col10/1733732175574/Put/seqid=0 2024-12-09T08:16:16,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742270_1446 (size=12104) 2024-12-09T08:16:16,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742271_1447 (size=12104) 2024-12-09T08:16:16,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T08:16:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:16,863 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T08:16:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:16,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/722c5602091f4d08b346cc037927c585 is 50, key is test_row_0/A:col10/1733732175624/Put/seqid=0 2024-12-09T08:16:16,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742272_1448 (size=12001) 2024-12-09T08:16:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T08:16:17,257 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/24bd5dc35dd74e21a2de0cff6ef3acf6 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/24bd5dc35dd74e21a2de0cff6ef3acf6 2024-12-09T08:16:17,261 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/B of 6b37acb92571896382ef9b774ba15dad into 24bd5dc35dd74e21a2de0cff6ef3acf6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:17,262 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:17,262 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/B, priority=13, startTime=1733732176841; duration=0sec 2024-12-09T08:16:17,262 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:17,262 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:B 2024-12-09T08:16:17,262 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:17,262 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/7ea2e25e868c4578a5f5d3f700a4969e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/7ea2e25e868c4578a5f5d3f700a4969e 2024-12-09T08:16:17,262 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:17,262 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/C is initiating minor compaction (all files) 2024-12-09T08:16:17,262 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/C in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:17,263 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/a70b67aa4bc9479c976dd199374a134c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/965a00fc0f104efcae1d15c325186c77, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/bd6d0546bd0f4871b60bcadd62a05df3] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=35.2 K 2024-12-09T08:16:17,263 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a70b67aa4bc9479c976dd199374a134c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733732173645 2024-12-09T08:16:17,263 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 965a00fc0f104efcae1d15c325186c77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733732173748 2024-12-09T08:16:17,263 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting bd6d0546bd0f4871b60bcadd62a05df3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733732175574 2024-12-09T08:16:17,266 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/A of 6b37acb92571896382ef9b774ba15dad into 7ea2e25e868c4578a5f5d3f700a4969e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:17,266 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:17,266 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/A, priority=13, startTime=1733732176841; duration=0sec 2024-12-09T08:16:17,266 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:17,266 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:A 2024-12-09T08:16:17,268 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:17,268 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/50e3d35865ee44c4872449957fa05eac is 50, key is test_row_0/C:col10/1733732175574/Put/seqid=0 2024-12-09T08:16:17,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742273_1449 (size=12104) 2024-12-09T08:16:17,280 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/722c5602091f4d08b346cc037927c585 2024-12-09T08:16:17,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/e40268c04efa47edb5a0d7a73f4fe67a is 50, key is test_row_0/B:col10/1733732175624/Put/seqid=0 2024-12-09T08:16:17,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742274_1450 (size=12001) 2024-12-09T08:16:17,675 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/50e3d35865ee44c4872449957fa05eac as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/50e3d35865ee44c4872449957fa05eac 2024-12-09T08:16:17,678 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/C of 6b37acb92571896382ef9b774ba15dad into 50e3d35865ee44c4872449957fa05eac(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:17,678 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:17,678 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/C, priority=13, startTime=1733732176841; duration=0sec 2024-12-09T08:16:17,679 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:17,679 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:C 2024-12-09T08:16:17,694 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/e40268c04efa47edb5a0d7a73f4fe67a 2024-12-09T08:16:17,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/710b1e44de3c4692a51f489c1a98038d is 50, key is test_row_0/C:col10/1733732175624/Put/seqid=0 2024-12-09T08:16:17,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742275_1451 (size=12001) 2024-12-09T08:16:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:17,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:17,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732237825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:17,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732237826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:17,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732237827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T08:16:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732237941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732237941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732237941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:17,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732237961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:17,964 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 6b37acb92571896382ef9b774ba15dad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:17,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732237971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:17,975 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 6b37acb92571896382ef9b774ba15dad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,103 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/710b1e44de3c4692a51f489c1a98038d 2024-12-09T08:16:18,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/722c5602091f4d08b346cc037927c585 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/722c5602091f4d08b346cc037927c585 2024-12-09T08:16:18,110 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/722c5602091f4d08b346cc037927c585, entries=150, sequenceid=80, filesize=11.7 K 2024-12-09T08:16:18,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/e40268c04efa47edb5a0d7a73f4fe67a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/e40268c04efa47edb5a0d7a73f4fe67a 2024-12-09T08:16:18,114 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/e40268c04efa47edb5a0d7a73f4fe67a, entries=150, sequenceid=80, filesize=11.7 K 2024-12-09T08:16:18,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/710b1e44de3c4692a51f489c1a98038d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/710b1e44de3c4692a51f489c1a98038d 2024-12-09T08:16:18,118 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/710b1e44de3c4692a51f489c1a98038d, entries=150, sequenceid=80, filesize=11.7 K 2024-12-09T08:16:18,119 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6b37acb92571896382ef9b774ba15dad in 1256ms, sequenceid=80, compaction requested=false 2024-12-09T08:16:18,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:18,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:18,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-09T08:16:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-09T08:16:18,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-09T08:16:18,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3270 sec 2024-12-09T08:16:18,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.3320 sec 2024-12-09T08:16:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:18,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T08:16:18,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:18,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:18,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:18,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:18,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:18,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:18,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/0ce71a89c7ff409096cac7b345fc72ec is 50, key is test_row_0/A:col10/1733732178156/Put/seqid=0 2024-12-09T08:16:18,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742276_1452 (size=19021) 2024-12-09T08:16:18,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732238186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:18,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732238194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:18,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732238194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732238301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732238310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:18,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732238311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732238511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:18,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732238521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:18,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732238521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:18,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/0ce71a89c7ff409096cac7b345fc72ec 2024-12-09T08:16:18,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/43d88e0d508945ffb73a648f5f0f3b86 is 50, key is test_row_0/B:col10/1733732178156/Put/seqid=0 2024-12-09T08:16:18,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742277_1453 (size=12001) 2024-12-09T08:16:18,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732238821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732238831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:18,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732238831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:18,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/43d88e0d508945ffb73a648f5f0f3b86 2024-12-09T08:16:18,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/73ddf6b58d0443bfb3b1fb8057751d9a is 50, key is test_row_0/C:col10/1733732178156/Put/seqid=0 2024-12-09T08:16:18,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742278_1454 (size=12001) 2024-12-09T08:16:19,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732239331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:19,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732239341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:19,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732239341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:19,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/73ddf6b58d0443bfb3b1fb8057751d9a 2024-12-09T08:16:19,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/0ce71a89c7ff409096cac7b345fc72ec as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/0ce71a89c7ff409096cac7b345fc72ec 2024-12-09T08:16:19,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/0ce71a89c7ff409096cac7b345fc72ec, entries=300, sequenceid=98, filesize=18.6 K 2024-12-09T08:16:19,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/43d88e0d508945ffb73a648f5f0f3b86 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/43d88e0d508945ffb73a648f5f0f3b86 2024-12-09T08:16:19,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/43d88e0d508945ffb73a648f5f0f3b86, entries=150, sequenceid=98, filesize=11.7 K 2024-12-09T08:16:19,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/73ddf6b58d0443bfb3b1fb8057751d9a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/73ddf6b58d0443bfb3b1fb8057751d9a 2024-12-09T08:16:19,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/73ddf6b58d0443bfb3b1fb8057751d9a, entries=150, sequenceid=98, filesize=11.7 K 2024-12-09T08:16:19,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6b37acb92571896382ef9b774ba15dad in 1243ms, sequenceid=98, compaction requested=true 2024-12-09T08:16:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:19,399 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:19,399 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:19,400 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:19,400 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43126 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:19,400 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/A is initiating minor compaction (all files) 2024-12-09T08:16:19,400 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/B is initiating minor compaction (all files) 2024-12-09T08:16:19,400 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/A in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:19,400 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/B in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:19,400 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/7ea2e25e868c4578a5f5d3f700a4969e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/722c5602091f4d08b346cc037927c585, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/0ce71a89c7ff409096cac7b345fc72ec] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=42.1 K 2024-12-09T08:16:19,400 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/24bd5dc35dd74e21a2de0cff6ef3acf6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/e40268c04efa47edb5a0d7a73f4fe67a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/43d88e0d508945ffb73a648f5f0f3b86] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=35.3 K 2024-12-09T08:16:19,400 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ea2e25e868c4578a5f5d3f700a4969e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733732175574 2024-12-09T08:16:19,400 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 24bd5dc35dd74e21a2de0cff6ef3acf6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733732175574 2024-12-09T08:16:19,400 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting e40268c04efa47edb5a0d7a73f4fe67a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733732175624 2024-12-09T08:16:19,400 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 722c5602091f4d08b346cc037927c585, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733732175624 2024-12-09T08:16:19,401 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 43d88e0d508945ffb73a648f5f0f3b86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733732177825 2024-12-09T08:16:19,401 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ce71a89c7ff409096cac7b345fc72ec, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733732177817 2024-12-09T08:16:19,408 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#B#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:19,408 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#A#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:19,409 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/1d3ba3fd3b96417a8ac96361ba767c2c is 50, key is test_row_0/B:col10/1733732178156/Put/seqid=0 2024-12-09T08:16:19,409 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/d3ae8216ec83442a8851b82eba8e44f7 is 50, key is test_row_0/A:col10/1733732178156/Put/seqid=0 2024-12-09T08:16:19,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742280_1456 (size=12207) 2024-12-09T08:16:19,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742279_1455 (size=12207) 2024-12-09T08:16:19,830 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/1d3ba3fd3b96417a8ac96361ba767c2c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/1d3ba3fd3b96417a8ac96361ba767c2c 2024-12-09T08:16:19,831 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/d3ae8216ec83442a8851b82eba8e44f7 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/d3ae8216ec83442a8851b82eba8e44f7 2024-12-09T08:16:19,837 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/A of 6b37acb92571896382ef9b774ba15dad into d3ae8216ec83442a8851b82eba8e44f7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:19,838 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:19,838 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/A, priority=13, startTime=1733732179399; duration=0sec 2024-12-09T08:16:19,838 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:19,838 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:A 2024-12-09T08:16:19,838 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:19,839 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/B of 6b37acb92571896382ef9b774ba15dad into 1d3ba3fd3b96417a8ac96361ba767c2c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:19,839 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:19,839 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/B, priority=13, startTime=1733732179399; duration=0sec 2024-12-09T08:16:19,839 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:19,839 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:B 2024-12-09T08:16:19,840 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:19,840 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/C is initiating minor compaction (all files) 2024-12-09T08:16:19,840 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/C in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:19,840 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/50e3d35865ee44c4872449957fa05eac, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/710b1e44de3c4692a51f489c1a98038d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/73ddf6b58d0443bfb3b1fb8057751d9a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=35.3 K 2024-12-09T08:16:19,840 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50e3d35865ee44c4872449957fa05eac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733732175574 2024-12-09T08:16:19,840 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 710b1e44de3c4692a51f489c1a98038d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733732175624 2024-12-09T08:16:19,841 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73ddf6b58d0443bfb3b1fb8057751d9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733732177825 2024-12-09T08:16:19,865 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#C#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:19,866 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/15b371b5cb884329a15e24798d634b35 is 50, key is test_row_0/C:col10/1733732178156/Put/seqid=0 2024-12-09T08:16:19,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742281_1457 (size=12207) 2024-12-09T08:16:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T08:16:19,952 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:19,953 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-09T08:16:19,956 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T08:16:19,958 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:19,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T08:16:20,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T08:16:20,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:20,111 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T08:16:20,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:20,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:20,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:20,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:20,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:20,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:20,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/bf5d451504fb4d7f89e15b8cc8001935 is 50, key is test_row_0/A:col10/1733732178193/Put/seqid=0 2024-12-09T08:16:20,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742282_1458 (size=12001) 2024-12-09T08:16:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T08:16:20,278 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/15b371b5cb884329a15e24798d634b35 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/15b371b5cb884329a15e24798d634b35 2024-12-09T08:16:20,294 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/C of 6b37acb92571896382ef9b774ba15dad into 15b371b5cb884329a15e24798d634b35(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:20,294 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:20,294 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/C, priority=13, startTime=1733732179399; duration=0sec 2024-12-09T08:16:20,294 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:20,294 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:C 2024-12-09T08:16:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:20,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:20,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732240399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:20,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732240399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:20,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732240402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:20,522 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/bf5d451504fb4d7f89e15b8cc8001935 2024-12-09T08:16:20,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732240521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732240521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732240521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:20,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/6a4e6e05b044459a8a4fece94eb9bbf7 is 50, key is test_row_0/B:col10/1733732178193/Put/seqid=0 2024-12-09T08:16:20,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742283_1459 (size=12001) 2024-12-09T08:16:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T08:16:20,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:20,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732240731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732240731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732240745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:20,956 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/6a4e6e05b044459a8a4fece94eb9bbf7 2024-12-09T08:16:20,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/4f51225fe0af4efba80d3aa148bfebcd is 50, key is test_row_0/C:col10/1733732178193/Put/seqid=0 2024-12-09T08:16:20,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742284_1460 (size=12001) 2024-12-09T08:16:20,983 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/4f51225fe0af4efba80d3aa148bfebcd 2024-12-09T08:16:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/bf5d451504fb4d7f89e15b8cc8001935 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/bf5d451504fb4d7f89e15b8cc8001935 2024-12-09T08:16:20,996 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/bf5d451504fb4d7f89e15b8cc8001935, entries=150, sequenceid=117, filesize=11.7 K 2024-12-09T08:16:20,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/6a4e6e05b044459a8a4fece94eb9bbf7 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/6a4e6e05b044459a8a4fece94eb9bbf7 2024-12-09T08:16:21,003 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/6a4e6e05b044459a8a4fece94eb9bbf7, entries=150, sequenceid=117, filesize=11.7 K 2024-12-09T08:16:21,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/4f51225fe0af4efba80d3aa148bfebcd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/4f51225fe0af4efba80d3aa148bfebcd 2024-12-09T08:16:21,011 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/4f51225fe0af4efba80d3aa148bfebcd, entries=150, sequenceid=117, filesize=11.7 K 2024-12-09T08:16:21,017 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6b37acb92571896382ef9b774ba15dad in 905ms, sequenceid=117, compaction requested=false 2024-12-09T08:16:21,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:21,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:21,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-09T08:16:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-09T08:16:21,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-09T08:16:21,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0630 sec 2024-12-09T08:16:21,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.0700 sec 2024-12-09T08:16:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:21,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T08:16:21,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:21,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:21,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:21,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:21,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:21,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:21,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/887b1d93f4f94ed88d50a2433ca0e7c5 is 50, key is test_row_0/A:col10/1733732180397/Put/seqid=0 2024-12-09T08:16:21,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742285_1461 (size=14491) 2024-12-09T08:16:21,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/887b1d93f4f94ed88d50a2433ca0e7c5 2024-12-09T08:16:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T08:16:21,092 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:21,092 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-09T08:16:21,097 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:21,099 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:21,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T08:16:21,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/a8acc6cd5fe249f7902694d75eb7658b is 50, key is test_row_0/B:col10/1733732180397/Put/seqid=0 2024-12-09T08:16:21,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732241099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732241101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742286_1462 (size=12101) 2024-12-09T08:16:21,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732241114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/a8acc6cd5fe249f7902694d75eb7658b 2024-12-09T08:16:21,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/15a994e13aaf4d50af567db1bef72efc is 50, key is test_row_0/C:col10/1733732180397/Put/seqid=0 2024-12-09T08:16:21,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742287_1463 (size=12101) 2024-12-09T08:16:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T08:16:21,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732241222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732241223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732241231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T08:16:21,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:21,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:21,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:21,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:21,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:21,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:21,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T08:16:21,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:21,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:21,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:21,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:21,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T08:16:21,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732241431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732241441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732241441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/15a994e13aaf4d50af567db1bef72efc 2024-12-09T08:16:21,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/887b1d93f4f94ed88d50a2433ca0e7c5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/887b1d93f4f94ed88d50a2433ca0e7c5 2024-12-09T08:16:21,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/887b1d93f4f94ed88d50a2433ca0e7c5, entries=200, sequenceid=134, filesize=14.2 K 2024-12-09T08:16:21,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/a8acc6cd5fe249f7902694d75eb7658b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/a8acc6cd5fe249f7902694d75eb7658b 2024-12-09T08:16:21,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T08:16:21,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:21,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:21,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:21,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:21,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:21,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/a8acc6cd5fe249f7902694d75eb7658b, entries=150, sequenceid=134, filesize=11.8 K 2024-12-09T08:16:21,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/15a994e13aaf4d50af567db1bef72efc as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/15a994e13aaf4d50af567db1bef72efc 2024-12-09T08:16:21,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/15a994e13aaf4d50af567db1bef72efc, entries=150, sequenceid=134, filesize=11.8 K 2024-12-09T08:16:21,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6b37acb92571896382ef9b774ba15dad in 510ms, sequenceid=134, compaction requested=true 2024-12-09T08:16:21,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:21,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:21,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:21,571 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:21,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:21,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:21,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:21,571 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:21,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:21,573 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:21,573 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:21,573 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/B is initiating minor compaction (all files) 2024-12-09T08:16:21,573 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/A is initiating minor compaction (all files) 2024-12-09T08:16:21,574 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/A in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:21,574 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/B in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:21,574 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/1d3ba3fd3b96417a8ac96361ba767c2c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/6a4e6e05b044459a8a4fece94eb9bbf7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/a8acc6cd5fe249f7902694d75eb7658b] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=35.5 K 2024-12-09T08:16:21,574 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/d3ae8216ec83442a8851b82eba8e44f7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/bf5d451504fb4d7f89e15b8cc8001935, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/887b1d93f4f94ed88d50a2433ca0e7c5] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=37.8 K 2024-12-09T08:16:21,574 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d3ba3fd3b96417a8ac96361ba767c2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733732177825 2024-12-09T08:16:21,574 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3ae8216ec83442a8851b82eba8e44f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733732177825 2024-12-09T08:16:21,574 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a4e6e05b044459a8a4fece94eb9bbf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733732178181 2024-12-09T08:16:21,574 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf5d451504fb4d7f89e15b8cc8001935, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733732178181 2024-12-09T08:16:21,574 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 887b1d93f4f94ed88d50a2433ca0e7c5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733732180374 2024-12-09T08:16:21,575 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a8acc6cd5fe249f7902694d75eb7658b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733732180374 2024-12-09T08:16:21,591 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#A#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:21,591 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/8ad983a64acf4df18a441fd9c12d723f is 50, key is test_row_0/A:col10/1733732180397/Put/seqid=0 2024-12-09T08:16:21,592 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:21,593 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/18653e23f962412897fb82ab8dc14c2c is 50, key is test_row_0/B:col10/1733732180397/Put/seqid=0 2024-12-09T08:16:21,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742288_1464 (size=12409) 2024-12-09T08:16:21,616 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/8ad983a64acf4df18a441fd9c12d723f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8ad983a64acf4df18a441fd9c12d723f 2024-12-09T08:16:21,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742289_1465 (size=12409) 2024-12-09T08:16:21,623 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/A of 6b37acb92571896382ef9b774ba15dad into 8ad983a64acf4df18a441fd9c12d723f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:21,623 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:21,623 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/A, priority=13, startTime=1733732181571; duration=0sec 2024-12-09T08:16:21,623 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:21,623 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:A 2024-12-09T08:16:21,624 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:21,625 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:21,625 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/C is initiating minor compaction (all files) 2024-12-09T08:16:21,625 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/C in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:21,625 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/15b371b5cb884329a15e24798d634b35, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/4f51225fe0af4efba80d3aa148bfebcd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/15a994e13aaf4d50af567db1bef72efc] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=35.5 K 2024-12-09T08:16:21,626 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15b371b5cb884329a15e24798d634b35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733732177825 2024-12-09T08:16:21,626 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f51225fe0af4efba80d3aa148bfebcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733732178181 2024-12-09T08:16:21,626 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15a994e13aaf4d50af567db1bef72efc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733732180374 2024-12-09T08:16:21,636 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#C#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:21,637 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/cf70fc2eacfd4ecf94a0cb1eb30a5844 is 50, key is test_row_0/C:col10/1733732180397/Put/seqid=0 2024-12-09T08:16:21,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742290_1466 (size=12409) 2024-12-09T08:16:21,673 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/cf70fc2eacfd4ecf94a0cb1eb30a5844 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/cf70fc2eacfd4ecf94a0cb1eb30a5844 2024-12-09T08:16:21,677 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/C of 6b37acb92571896382ef9b774ba15dad into cf70fc2eacfd4ecf94a0cb1eb30a5844(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:21,677 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:21,677 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/C, priority=13, startTime=1733732181571; duration=0sec 2024-12-09T08:16:21,677 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:21,677 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:C 2024-12-09T08:16:21,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T08:16:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:21,713 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T08:16:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:21,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/bb2a2a7a45a24b0da0f559f3055365b4 is 50, key is test_row_0/A:col10/1733732181111/Put/seqid=0 2024-12-09T08:16:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T08:16:21,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742291_1467 (size=12151) 2024-12-09T08:16:21,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:21,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732241784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732241785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732241787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732241901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732241901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732241901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732241981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,991 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 6b37acb92571896382ef9b774ba15dad of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:21,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732241991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:21,998 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 6b37acb92571896382ef9b774ba15dad of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:22,024 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/18653e23f962412897fb82ab8dc14c2c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/18653e23f962412897fb82ab8dc14c2c 2024-12-09T08:16:22,033 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/B of 6b37acb92571896382ef9b774ba15dad into 18653e23f962412897fb82ab8dc14c2c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:22,033 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:22,033 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/B, priority=13, startTime=1733732181571; duration=0sec 2024-12-09T08:16:22,033 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:22,033 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:B 2024-12-09T08:16:22,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732242111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:22,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732242111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:22,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732242111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:22,138 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/bb2a2a7a45a24b0da0f559f3055365b4 2024-12-09T08:16:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/03c7aa4ad85d46fca89f7be4a9605320 is 50, key is test_row_0/B:col10/1733732181111/Put/seqid=0 2024-12-09T08:16:22,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742292_1468 (size=12151) 2024-12-09T08:16:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T08:16:22,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732242421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:22,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732242421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:22,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732242421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:22,548 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/03c7aa4ad85d46fca89f7be4a9605320 2024-12-09T08:16:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/ef087876310d4580b72a8c3a0e02432c is 50, key is test_row_0/C:col10/1733732181111/Put/seqid=0 2024-12-09T08:16:22,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742293_1469 (size=12151) 2024-12-09T08:16:22,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732242932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:22,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732242932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732242932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:22,958 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/ef087876310d4580b72a8c3a0e02432c 2024-12-09T08:16:22,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/bb2a2a7a45a24b0da0f559f3055365b4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/bb2a2a7a45a24b0da0f559f3055365b4 2024-12-09T08:16:22,978 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/bb2a2a7a45a24b0da0f559f3055365b4, entries=150, sequenceid=152, filesize=11.9 K 2024-12-09T08:16:22,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/03c7aa4ad85d46fca89f7be4a9605320 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/03c7aa4ad85d46fca89f7be4a9605320 2024-12-09T08:16:22,983 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/03c7aa4ad85d46fca89f7be4a9605320, entries=150, sequenceid=152, filesize=11.9 K 2024-12-09T08:16:22,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/ef087876310d4580b72a8c3a0e02432c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ef087876310d4580b72a8c3a0e02432c 2024-12-09T08:16:22,990 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ef087876310d4580b72a8c3a0e02432c, entries=150, sequenceid=152, filesize=11.9 K 2024-12-09T08:16:22,991 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6b37acb92571896382ef9b774ba15dad in 1278ms, sequenceid=152, compaction requested=false 2024-12-09T08:16:22,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:22,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:22,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-09T08:16:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-09T08:16:22,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-09T08:16:22,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8940 sec 2024-12-09T08:16:23,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.9070 sec 2024-12-09T08:16:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T08:16:23,242 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:23,242 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-09T08:16:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T08:16:23,244 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:23,244 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:23,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T08:16:23,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T08:16:23,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:23,396 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T08:16:23,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:23,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:23,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:23,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:23,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:23,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:23,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/890227c4ab734459bf8e3dd90fc4eb72 is 50, key is test_row_0/A:col10/1733732181783/Put/seqid=0 2024-12-09T08:16:23,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742294_1470 (size=12151) 2024-12-09T08:16:23,404 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/890227c4ab734459bf8e3dd90fc4eb72 2024-12-09T08:16:23,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/2f2361340cde46f5aa856bc47594700c is 50, key is test_row_0/B:col10/1733732181783/Put/seqid=0 2024-12-09T08:16:23,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742295_1471 (size=12151) 2024-12-09T08:16:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T08:16:23,814 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/2f2361340cde46f5aa856bc47594700c 2024-12-09T08:16:23,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/b9ace48ca7d6488ba59bf7ff676c92db is 50, key is test_row_0/C:col10/1733732181783/Put/seqid=0 2024-12-09T08:16:23,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742296_1472 (size=12151) 2024-12-09T08:16:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T08:16:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:23,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:23,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732243984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:23,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732243985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732243993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732244101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732244101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732244111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,228 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/b9ace48ca7d6488ba59bf7ff676c92db 2024-12-09T08:16:24,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/890227c4ab734459bf8e3dd90fc4eb72 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/890227c4ab734459bf8e3dd90fc4eb72 2024-12-09T08:16:24,235 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/890227c4ab734459bf8e3dd90fc4eb72, entries=150, sequenceid=169, filesize=11.9 K 2024-12-09T08:16:24,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/2f2361340cde46f5aa856bc47594700c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2f2361340cde46f5aa856bc47594700c 2024-12-09T08:16:24,242 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2f2361340cde46f5aa856bc47594700c, entries=150, sequenceid=169, filesize=11.9 K 2024-12-09T08:16:24,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/b9ace48ca7d6488ba59bf7ff676c92db as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/b9ace48ca7d6488ba59bf7ff676c92db 2024-12-09T08:16:24,247 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/b9ace48ca7d6488ba59bf7ff676c92db, entries=150, sequenceid=169, filesize=11.9 K 2024-12-09T08:16:24,247 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6b37acb92571896382ef9b774ba15dad in 851ms, sequenceid=169, compaction requested=true 2024-12-09T08:16:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-09T08:16:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-09T08:16:24,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-09T08:16:24,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0050 sec 2024-12-09T08:16:24,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.0110 sec 2024-12-09T08:16:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:24,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T08:16:24,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:24,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:24,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:24,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:24,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:24,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:24,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/7e677ef92aa9495d988e8744b4f27bff is 50, key is test_row_0/A:col10/1733732183984/Put/seqid=0 2024-12-09T08:16:24,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742297_1473 (size=16931) 2024-12-09T08:16:24,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/7e677ef92aa9495d988e8744b4f27bff 2024-12-09T08:16:24,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/8fa46eba8894430883d9f2e01ca8509b is 50, key is test_row_0/B:col10/1733732183984/Put/seqid=0 2024-12-09T08:16:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T08:16:24,384 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:24,384 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-09T08:16:24,386 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:24,387 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:24,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T08:16:24,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742298_1474 (size=12151) 2024-12-09T08:16:24,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/8fa46eba8894430883d9f2e01ca8509b 2024-12-09T08:16:24,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732244385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732244385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732244401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/3f35cad63b854b77af8b064a492fa845 is 50, key is test_row_0/C:col10/1733732183984/Put/seqid=0 2024-12-09T08:16:24,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742299_1475 (size=12151) 2024-12-09T08:16:24,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/3f35cad63b854b77af8b064a492fa845 2024-12-09T08:16:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/7e677ef92aa9495d988e8744b4f27bff as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/7e677ef92aa9495d988e8744b4f27bff 2024-12-09T08:16:24,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/7e677ef92aa9495d988e8744b4f27bff, entries=250, sequenceid=187, filesize=16.5 K 2024-12-09T08:16:24,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/8fa46eba8894430883d9f2e01ca8509b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/8fa46eba8894430883d9f2e01ca8509b 2024-12-09T08:16:24,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/8fa46eba8894430883d9f2e01ca8509b, entries=150, sequenceid=187, filesize=11.9 K 2024-12-09T08:16:24,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/3f35cad63b854b77af8b064a492fa845 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/3f35cad63b854b77af8b064a492fa845 2024-12-09T08:16:24,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/3f35cad63b854b77af8b064a492fa845, entries=150, sequenceid=187, filesize=11.9 K 2024-12-09T08:16:24,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6b37acb92571896382ef9b774ba15dad in 130ms, sequenceid=187, compaction requested=true 2024-12-09T08:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:24,459 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:24,459 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:24,460 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53642 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:16:24,460 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:16:24,460 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/A is initiating minor compaction (all files) 2024-12-09T08:16:24,460 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/B is initiating minor compaction (all files) 2024-12-09T08:16:24,460 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/A in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:24,460 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/B in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:24,460 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/18653e23f962412897fb82ab8dc14c2c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/03c7aa4ad85d46fca89f7be4a9605320, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2f2361340cde46f5aa856bc47594700c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/8fa46eba8894430883d9f2e01ca8509b] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=47.7 K 2024-12-09T08:16:24,460 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8ad983a64acf4df18a441fd9c12d723f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/bb2a2a7a45a24b0da0f559f3055365b4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/890227c4ab734459bf8e3dd90fc4eb72, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/7e677ef92aa9495d988e8744b4f27bff] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=52.4 K 2024-12-09T08:16:24,465 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 18653e23f962412897fb82ab8dc14c2c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733732180374 2024-12-09T08:16:24,465 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c7aa4ad85d46fca89f7be4a9605320, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733732181075 2024-12-09T08:16:24,465 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad983a64acf4df18a441fd9c12d723f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733732180374 2024-12-09T08:16:24,466 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f2361340cde46f5aa856bc47594700c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733732181782 2024-12-09T08:16:24,466 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb2a2a7a45a24b0da0f559f3055365b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733732181075 2024-12-09T08:16:24,467 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 890227c4ab734459bf8e3dd90fc4eb72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733732181782 2024-12-09T08:16:24,467 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fa46eba8894430883d9f2e01ca8509b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733732183984 2024-12-09T08:16:24,467 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e677ef92aa9495d988e8744b4f27bff, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733732183969 2024-12-09T08:16:24,481 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#B#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:24,481 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/9820b7857ffc4e978297765724e17fe6 is 50, key is test_row_0/B:col10/1733732183984/Put/seqid=0 2024-12-09T08:16:24,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742300_1476 (size=12595) 2024-12-09T08:16:24,487 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:24,487 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/ba1cc4774a3b47d0a78ef06c177260d6 is 50, key is test_row_0/A:col10/1733732183984/Put/seqid=0 2024-12-09T08:16:24,492 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/9820b7857ffc4e978297765724e17fe6 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/9820b7857ffc4e978297765724e17fe6 2024-12-09T08:16:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T08:16:24,497 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/B of 6b37acb92571896382ef9b774ba15dad into 9820b7857ffc4e978297765724e17fe6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:24,497 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:24,497 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/B, priority=12, startTime=1733732184459; duration=0sec 2024-12-09T08:16:24,497 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:24,497 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:B 2024-12-09T08:16:24,497 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:16:24,498 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:16:24,498 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/C is initiating minor compaction (all files) 2024-12-09T08:16:24,498 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/C in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:24,498 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/cf70fc2eacfd4ecf94a0cb1eb30a5844, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ef087876310d4580b72a8c3a0e02432c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/b9ace48ca7d6488ba59bf7ff676c92db, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/3f35cad63b854b77af8b064a492fa845] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=47.7 K 2024-12-09T08:16:24,498 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting cf70fc2eacfd4ecf94a0cb1eb30a5844, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733732180374 2024-12-09T08:16:24,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742301_1477 (size=12595) 2024-12-09T08:16:24,499 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ef087876310d4580b72a8c3a0e02432c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733732181075 2024-12-09T08:16:24,499 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting b9ace48ca7d6488ba59bf7ff676c92db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733732181782 2024-12-09T08:16:24,499 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f35cad63b854b77af8b064a492fa845, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733732183984 2024-12-09T08:16:24,506 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#C#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:24,506 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/f21627a57653436281a11171075a17ef is 50, key is test_row_0/C:col10/1733732183984/Put/seqid=0 2024-12-09T08:16:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:24,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T08:16:24,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:24,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:24,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:24,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:24,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:24,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:24,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742302_1478 (size=12595) 2024-12-09T08:16:24,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/32c2a12fe88b4e8fb52f98d77877ebea is 50, key is test_row_0/A:col10/1733732184514/Put/seqid=0 2024-12-09T08:16:24,528 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/f21627a57653436281a11171075a17ef as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/f21627a57653436281a11171075a17ef 2024-12-09T08:16:24,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742303_1479 (size=14541) 2024-12-09T08:16:24,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/32c2a12fe88b4e8fb52f98d77877ebea 2024-12-09T08:16:24,533 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/C of 6b37acb92571896382ef9b774ba15dad into f21627a57653436281a11171075a17ef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:24,533 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:24,533 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/C, priority=12, startTime=1733732184459; duration=0sec 2024-12-09T08:16:24,533 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:24,533 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:C 2024-12-09T08:16:24,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T08:16:24,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:24,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:24,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:24,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:24,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:24,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/9f5877d375cc402c89a5332c83b2b730 is 50, key is test_row_0/B:col10/1733732184514/Put/seqid=0 2024-12-09T08:16:24,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742304_1480 (size=12151) 2024-12-09T08:16:24,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732244562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732244565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732244571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732244681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732244681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T08:16:24,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:24,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:24,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:24,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:24,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:24,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732244691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T08:16:24,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T08:16:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:24,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:24,904 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/ba1cc4774a3b47d0a78ef06c177260d6 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/ba1cc4774a3b47d0a78ef06c177260d6 2024-12-09T08:16:24,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732244901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732244901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,913 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/A of 6b37acb92571896382ef9b774ba15dad into ba1cc4774a3b47d0a78ef06c177260d6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:24,913 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:24,913 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/A, priority=12, startTime=1733732184459; duration=0sec 2024-12-09T08:16:24,913 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:24,913 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:A 2024-12-09T08:16:24,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732244911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:24,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/9f5877d375cc402c89a5332c83b2b730 2024-12-09T08:16:24,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/ecc149855e3341d5ae8be45b00efa927 is 50, key is test_row_0/C:col10/1733732184514/Put/seqid=0 2024-12-09T08:16:24,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742305_1481 (size=12151) 2024-12-09T08:16:24,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/ecc149855e3341d5ae8be45b00efa927 2024-12-09T08:16:24,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/32c2a12fe88b4e8fb52f98d77877ebea as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/32c2a12fe88b4e8fb52f98d77877ebea 2024-12-09T08:16:24,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/32c2a12fe88b4e8fb52f98d77877ebea, entries=200, sequenceid=203, filesize=14.2 K 2024-12-09T08:16:24,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/9f5877d375cc402c89a5332c83b2b730 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/9f5877d375cc402c89a5332c83b2b730 2024-12-09T08:16:24,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/9f5877d375cc402c89a5332c83b2b730, entries=150, sequenceid=203, filesize=11.9 K 2024-12-09T08:16:24,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/ecc149855e3341d5ae8be45b00efa927 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ecc149855e3341d5ae8be45b00efa927 2024-12-09T08:16:24,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ecc149855e3341d5ae8be45b00efa927, entries=150, sequenceid=203, filesize=11.9 K 2024-12-09T08:16:24,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6b37acb92571896382ef9b774ba15dad in 458ms, sequenceid=203, compaction requested=false 2024-12-09T08:16:24,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:25,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T08:16:25,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:25,001 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T08:16:25,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:25,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:25,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:25,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:25,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:25,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:25,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/de7ae6c078f84d7ab8687df4d888d2b4 is 50, key is test_row_0/A:col10/1733732184560/Put/seqid=0 2024-12-09T08:16:25,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742306_1482 (size=12151) 2024-12-09T08:16:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T08:16:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:25,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:25,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732245240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:25,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732245248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:25,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:25,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732245251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732245361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:25,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732245361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732245361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:25,408 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/de7ae6c078f84d7ab8687df4d888d2b4 2024-12-09T08:16:25,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/7120c3e4d2fa40f6a7fe8c016583d73c is 50, key is test_row_0/B:col10/1733732184560/Put/seqid=0 2024-12-09T08:16:25,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742307_1483 (size=12151) 2024-12-09T08:16:25,416 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/7120c3e4d2fa40f6a7fe8c016583d73c 2024-12-09T08:16:25,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/1e70114aba5b448cac57cce603da9999 is 50, key is test_row_0/C:col10/1733732184560/Put/seqid=0 2024-12-09T08:16:25,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742308_1484 (size=12151) 2024-12-09T08:16:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T08:16:25,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732245571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732245571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:25,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732245581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:25,824 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/1e70114aba5b448cac57cce603da9999 2024-12-09T08:16:25,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/de7ae6c078f84d7ab8687df4d888d2b4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/de7ae6c078f84d7ab8687df4d888d2b4 2024-12-09T08:16:25,830 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/de7ae6c078f84d7ab8687df4d888d2b4, entries=150, sequenceid=222, filesize=11.9 K 2024-12-09T08:16:25,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/7120c3e4d2fa40f6a7fe8c016583d73c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/7120c3e4d2fa40f6a7fe8c016583d73c 2024-12-09T08:16:25,833 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/7120c3e4d2fa40f6a7fe8c016583d73c, entries=150, sequenceid=222, filesize=11.9 K 2024-12-09T08:16:25,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/1e70114aba5b448cac57cce603da9999 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/1e70114aba5b448cac57cce603da9999 2024-12-09T08:16:25,836 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/1e70114aba5b448cac57cce603da9999, entries=150, sequenceid=222, filesize=11.9 K 2024-12-09T08:16:25,837 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6b37acb92571896382ef9b774ba15dad in 837ms, sequenceid=222, compaction requested=true 2024-12-09T08:16:25,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:25,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:25,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-09T08:16:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-09T08:16:25,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-09T08:16:25,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4510 sec 2024-12-09T08:16:25,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4560 sec 2024-12-09T08:16:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:25,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T08:16:25,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:25,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:25,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:25,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:25,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:25,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:25,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/88e1cb948f1a4d33a84ffbb0ae4243de is 50, key is test_row_0/A:col10/1733732185885/Put/seqid=0 2024-12-09T08:16:25,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742309_1485 (size=14541) 2024-12-09T08:16:25,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732245920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:25,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732245925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:25,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732245926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:25,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:25,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:26,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732246031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732246031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:26,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732246041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:26,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732246241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:26,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732246241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:26,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732246261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:26,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/88e1cb948f1a4d33a84ffbb0ae4243de 2024-12-09T08:16:26,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/cdd82597b78c4e9cb9378b8e0178efe6 is 50, key is test_row_0/B:col10/1733732185885/Put/seqid=0 2024-12-09T08:16:26,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742310_1486 (size=12151) 2024-12-09T08:16:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T08:16:26,531 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:26,531 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-09T08:16:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T08:16:26,533 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:26,533 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:26,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:26,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732246551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:26,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:26,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732246551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:26,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:26,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:26,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732246581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T08:16:26,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T08:16:26,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:26,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:26,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:26,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:26,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:26,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/cdd82597b78c4e9cb9378b8e0178efe6 2024-12-09T08:16:26,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/c0edb4315d3b4ac497c5fc2a73955a50 is 50, key is test_row_0/C:col10/1733732185885/Put/seqid=0 2024-12-09T08:16:26,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742311_1487 (size=12151) 2024-12-09T08:16:26,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T08:16:26,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:26,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:26,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:26,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:26,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T08:16:26,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T08:16:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:26,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:27,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732247061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:27,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732247071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:27,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:27,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:27,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:27,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732247091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:27,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/c0edb4315d3b4ac497c5fc2a73955a50 2024-12-09T08:16:27,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/88e1cb948f1a4d33a84ffbb0ae4243de as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/88e1cb948f1a4d33a84ffbb0ae4243de 2024-12-09T08:16:27,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/88e1cb948f1a4d33a84ffbb0ae4243de, entries=200, sequenceid=238, filesize=14.2 K 2024-12-09T08:16:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/cdd82597b78c4e9cb9378b8e0178efe6 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/cdd82597b78c4e9cb9378b8e0178efe6 2024-12-09T08:16:27,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/cdd82597b78c4e9cb9378b8e0178efe6, entries=150, sequenceid=238, filesize=11.9 K 2024-12-09T08:16:27,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/c0edb4315d3b4ac497c5fc2a73955a50 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/c0edb4315d3b4ac497c5fc2a73955a50 2024-12-09T08:16:27,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/c0edb4315d3b4ac497c5fc2a73955a50, entries=150, sequenceid=238, filesize=11.9 K 2024-12-09T08:16:27,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6b37acb92571896382ef9b774ba15dad in 1257ms, sequenceid=238, compaction requested=true 2024-12-09T08:16:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:27,143 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:16:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:27,143 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:16:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:27,144 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:16:27,144 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:16:27,144 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/A is initiating minor compaction (all files) 2024-12-09T08:16:27,144 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/B is initiating minor compaction (all files) 2024-12-09T08:16:27,144 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/A in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:27,144 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/B in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:27,144 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/ba1cc4774a3b47d0a78ef06c177260d6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/32c2a12fe88b4e8fb52f98d77877ebea, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/de7ae6c078f84d7ab8687df4d888d2b4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/88e1cb948f1a4d33a84ffbb0ae4243de] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=52.6 K 2024-12-09T08:16:27,144 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/9820b7857ffc4e978297765724e17fe6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/9f5877d375cc402c89a5332c83b2b730, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/7120c3e4d2fa40f6a7fe8c016583d73c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/cdd82597b78c4e9cb9378b8e0178efe6] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=47.9 K 2024-12-09T08:16:27,145 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba1cc4774a3b47d0a78ef06c177260d6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733732183984 2024-12-09T08:16:27,145 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9820b7857ffc4e978297765724e17fe6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733732183984 2024-12-09T08:16:27,145 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32c2a12fe88b4e8fb52f98d77877ebea, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733732184338 2024-12-09T08:16:27,145 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f5877d375cc402c89a5332c83b2b730, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733732184380 2024-12-09T08:16:27,146 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting de7ae6c078f84d7ab8687df4d888d2b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733732184560 2024-12-09T08:16:27,146 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88e1cb948f1a4d33a84ffbb0ae4243de, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733732185238 2024-12-09T08:16:27,146 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 7120c3e4d2fa40f6a7fe8c016583d73c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733732184560 2024-12-09T08:16:27,146 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd82597b78c4e9cb9378b8e0178efe6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733732185246 2024-12-09T08:16:27,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T08:16:27,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:27,148 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T08:16:27,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:27,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:27,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:27,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:27,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:27,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T08:16:27,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/f4e1562ea7964d09905b225bef1499de is 50, key is test_row_0/A:col10/1733732185924/Put/seqid=0 2024-12-09T08:16:27,162 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:27,163 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/c3dacf4284d9422c914d1ab39f3346cd is 50, key is test_row_0/B:col10/1733732185885/Put/seqid=0 2024-12-09T08:16:27,166 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#A#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:27,167 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/b51cf2c3f07f466182666cf47bb7da31 is 50, key is test_row_0/A:col10/1733732185885/Put/seqid=0 2024-12-09T08:16:27,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742313_1489 (size=12731) 2024-12-09T08:16:27,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742312_1488 (size=12151) 2024-12-09T08:16:27,190 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/f4e1562ea7964d09905b225bef1499de 2024-12-09T08:16:27,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/2481e1c30397498aa5bdd5ce5834e34a is 50, key is test_row_0/B:col10/1733732185924/Put/seqid=0 2024-12-09T08:16:27,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742314_1490 (size=12731) 2024-12-09T08:16:27,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742315_1491 (size=12151) 2024-12-09T08:16:27,598 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/c3dacf4284d9422c914d1ab39f3346cd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/c3dacf4284d9422c914d1ab39f3346cd 2024-12-09T08:16:27,604 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/2481e1c30397498aa5bdd5ce5834e34a 2024-12-09T08:16:27,606 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/b51cf2c3f07f466182666cf47bb7da31 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/b51cf2c3f07f466182666cf47bb7da31 2024-12-09T08:16:27,611 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/B of 6b37acb92571896382ef9b774ba15dad into c3dacf4284d9422c914d1ab39f3346cd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:27,611 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:27,611 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/B, priority=12, startTime=1733732187143; duration=0sec 2024-12-09T08:16:27,611 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:27,611 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:B 2024-12-09T08:16:27,611 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:16:27,619 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/A of 6b37acb92571896382ef9b774ba15dad into b51cf2c3f07f466182666cf47bb7da31(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:27,619 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:27,619 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/A, priority=12, startTime=1733732187143; duration=0sec 2024-12-09T08:16:27,619 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:27,619 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:A 2024-12-09T08:16:27,620 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:16:27,620 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/C is initiating minor compaction (all files) 2024-12-09T08:16:27,620 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/C in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:27,620 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/f21627a57653436281a11171075a17ef, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ecc149855e3341d5ae8be45b00efa927, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/1e70114aba5b448cac57cce603da9999, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/c0edb4315d3b4ac497c5fc2a73955a50] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=47.9 K 2024-12-09T08:16:27,621 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting f21627a57653436281a11171075a17ef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733732183984 2024-12-09T08:16:27,621 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ecc149855e3341d5ae8be45b00efa927, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733732184380 2024-12-09T08:16:27,621 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e70114aba5b448cac57cce603da9999, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733732184560 2024-12-09T08:16:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/82ae508119f24f35a87627fd9d8d4dcb is 50, key is test_row_0/C:col10/1733732185924/Put/seqid=0 2024-12-09T08:16:27,622 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting c0edb4315d3b4ac497c5fc2a73955a50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733732185246 2024-12-09T08:16:27,630 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#C#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:27,631 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/4e92be2570974c699a72cfcb9e703b7c is 50, key is test_row_0/C:col10/1733732185885/Put/seqid=0 2024-12-09T08:16:27,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742316_1492 (size=12731) 2024-12-09T08:16:27,651 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/4e92be2570974c699a72cfcb9e703b7c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/4e92be2570974c699a72cfcb9e703b7c 2024-12-09T08:16:27,669 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/C of 6b37acb92571896382ef9b774ba15dad into 4e92be2570974c699a72cfcb9e703b7c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:27,669 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:27,669 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/C, priority=12, startTime=1733732187143; duration=0sec 2024-12-09T08:16:27,669 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:27,669 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:C 2024-12-09T08:16:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T08:16:27,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742317_1493 (size=12151) 2024-12-09T08:16:27,682 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/82ae508119f24f35a87627fd9d8d4dcb 2024-12-09T08:16:27,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/f4e1562ea7964d09905b225bef1499de as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/f4e1562ea7964d09905b225bef1499de 2024-12-09T08:16:27,692 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/f4e1562ea7964d09905b225bef1499de, entries=150, sequenceid=254, filesize=11.9 K 2024-12-09T08:16:27,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/2481e1c30397498aa5bdd5ce5834e34a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2481e1c30397498aa5bdd5ce5834e34a 2024-12-09T08:16:27,699 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2481e1c30397498aa5bdd5ce5834e34a, entries=150, sequenceid=254, filesize=11.9 K 2024-12-09T08:16:27,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/82ae508119f24f35a87627fd9d8d4dcb as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/82ae508119f24f35a87627fd9d8d4dcb 2024-12-09T08:16:27,710 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/82ae508119f24f35a87627fd9d8d4dcb, entries=150, sequenceid=254, filesize=11.9 K 2024-12-09T08:16:27,713 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 6b37acb92571896382ef9b774ba15dad in 565ms, sequenceid=254, compaction requested=false 2024-12-09T08:16:27,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:27,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:27,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-09T08:16:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-09T08:16:27,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-09T08:16:27,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1810 sec 2024-12-09T08:16:27,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.1890 sec 2024-12-09T08:16:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:28,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:16:28,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:28,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:28,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:28,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:28,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:28,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:28,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/5e5eec2894ca40e4ba1a370ff9d940c1 is 50, key is test_row_0/A:col10/1733732188083/Put/seqid=0 2024-12-09T08:16:28,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742318_1494 (size=12297) 2024-12-09T08:16:28,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732188122, max timestamp=1733732188137Num uniques 859; 2024-12-09T08:16:28,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:16:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732248132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732188122, max timestamp=1733732188137Num uniques 859; 2024-12-09T08:16:28,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:16:28,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732188122, max timestamp=1733732188137Num uniques 859; 2024-12-09T08:16:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:16:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732248191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732248199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732248251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732248311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732248311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732248461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/5e5eec2894ca40e4ba1a370ff9d940c1 2024-12-09T08:16:28,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/3020f73b7749452ea5a0e54494e4b5ff is 50, key is test_row_0/B:col10/1733732188083/Put/seqid=0 2024-12-09T08:16:28,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742319_1495 (size=9857) 2024-12-09T08:16:28,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/3020f73b7749452ea5a0e54494e4b5ff 2024-12-09T08:16:28,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/252721d79e294c0eba752af9bf0fd649 is 50, key is test_row_0/C:col10/1733732188083/Put/seqid=0 2024-12-09T08:16:28,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742320_1496 (size=9857) 2024-12-09T08:16:28,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/252721d79e294c0eba752af9bf0fd649 2024-12-09T08:16:28,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/5e5eec2894ca40e4ba1a370ff9d940c1 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/5e5eec2894ca40e4ba1a370ff9d940c1 2024-12-09T08:16:28,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732248521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732248522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/5e5eec2894ca40e4ba1a370ff9d940c1, entries=150, sequenceid=267, filesize=12.0 K 2024-12-09T08:16:28,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/3020f73b7749452ea5a0e54494e4b5ff as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/3020f73b7749452ea5a0e54494e4b5ff 2024-12-09T08:16:28,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/3020f73b7749452ea5a0e54494e4b5ff, entries=100, sequenceid=267, filesize=9.6 K 2024-12-09T08:16:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/252721d79e294c0eba752af9bf0fd649 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/252721d79e294c0eba752af9bf0fd649 2024-12-09T08:16:28,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/252721d79e294c0eba752af9bf0fd649, entries=100, sequenceid=267, filesize=9.6 K 2024-12-09T08:16:28,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6b37acb92571896382ef9b774ba15dad in 457ms, sequenceid=267, compaction requested=true 2024-12-09T08:16:28,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:28,544 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:28,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:28,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:28,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:28,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:28,544 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:28,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:28,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:28,545 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37179 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:28,545 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/A is initiating minor compaction (all files) 2024-12-09T08:16:28,545 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/A in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:28,545 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/b51cf2c3f07f466182666cf47bb7da31, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/f4e1562ea7964d09905b225bef1499de, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/5e5eec2894ca40e4ba1a370ff9d940c1] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=36.3 K 2024-12-09T08:16:28,545 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:28,545 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/B is initiating minor compaction (all files) 2024-12-09T08:16:28,545 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/B in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:28,545 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/c3dacf4284d9422c914d1ab39f3346cd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2481e1c30397498aa5bdd5ce5834e34a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/3020f73b7749452ea5a0e54494e4b5ff] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=33.9 K 2024-12-09T08:16:28,547 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting c3dacf4284d9422c914d1ab39f3346cd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733732185246 2024-12-09T08:16:28,547 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b51cf2c3f07f466182666cf47bb7da31, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733732185246 2024-12-09T08:16:28,548 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2481e1c30397498aa5bdd5ce5834e34a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733732185919 2024-12-09T08:16:28,548 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4e1562ea7964d09905b225bef1499de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733732185919 2024-12-09T08:16:28,549 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e5eec2894ca40e4ba1a370ff9d940c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733732188080 2024-12-09T08:16:28,549 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3020f73b7749452ea5a0e54494e4b5ff, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733732188083 2024-12-09T08:16:28,562 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#A#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:28,562 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/1b35715f95f6428bb5c6b5eba1fe5aa1 is 50, key is test_row_0/A:col10/1733732188083/Put/seqid=0 2024-12-09T08:16:28,574 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:28,575 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/a44e0c2bb205403d8660f05be82a5068 is 50, key is test_row_0/B:col10/1733732188083/Put/seqid=0 2024-12-09T08:16:28,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742321_1497 (size=12933) 2024-12-09T08:16:28,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742322_1498 (size=12933) 2024-12-09T08:16:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T08:16:28,681 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:28,682 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-09T08:16:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T08:16:28,684 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:28,685 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:28,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:28,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-09T08:16:28,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:28,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:28,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:28,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:28,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:28,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T08:16:28,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/8ae6c82c260f4fa286e724b5fd283954 is 50, key is test_row_0/A:col10/1733732188197/Put/seqid=0 2024-12-09T08:16:28,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742323_1499 (size=14741) 2024-12-09T08:16:28,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T08:16:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:28,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:28,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732248841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732248841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732248856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732248961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732248981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:28,987 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/1b35715f95f6428bb5c6b5eba1fe5aa1 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/1b35715f95f6428bb5c6b5eba1fe5aa1 2024-12-09T08:16:28,991 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/A of 6b37acb92571896382ef9b774ba15dad into 1b35715f95f6428bb5c6b5eba1fe5aa1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:28,991 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:28,991 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/A, priority=13, startTime=1733732188543; duration=0sec 2024-12-09T08:16:28,991 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:28,991 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:A 2024-12-09T08:16:28,991 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:28,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T08:16:28,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:28,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:28,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:28,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:28,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:28,995 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:28,995 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/C is initiating minor compaction (all files) 2024-12-09T08:16:28,996 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/C in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:28,996 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/4e92be2570974c699a72cfcb9e703b7c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/82ae508119f24f35a87627fd9d8d4dcb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/252721d79e294c0eba752af9bf0fd649] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=33.9 K 2024-12-09T08:16:28,997 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/a44e0c2bb205403d8660f05be82a5068 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/a44e0c2bb205403d8660f05be82a5068 2024-12-09T08:16:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T08:16:29,001 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e92be2570974c699a72cfcb9e703b7c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733732185246 2024-12-09T08:16:29,002 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82ae508119f24f35a87627fd9d8d4dcb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733732185919 2024-12-09T08:16:29,003 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 252721d79e294c0eba752af9bf0fd649, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733732188083 2024-12-09T08:16:29,005 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/B of 6b37acb92571896382ef9b774ba15dad into a44e0c2bb205403d8660f05be82a5068(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:29,005 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:29,005 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/B, priority=13, startTime=1733732188544; duration=0sec 2024-12-09T08:16:29,005 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:29,005 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:B 2024-12-09T08:16:29,012 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#C#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:29,013 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/a21c0c3357a143719aa32053a6b25f13 is 50, key is test_row_0/C:col10/1733732188083/Put/seqid=0 2024-12-09T08:16:29,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742324_1500 (size=12933) 2024-12-09T08:16:29,028 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/a21c0c3357a143719aa32053a6b25f13 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/a21c0c3357a143719aa32053a6b25f13 2024-12-09T08:16:29,031 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/C of 6b37acb92571896382ef9b774ba15dad into a21c0c3357a143719aa32053a6b25f13(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:29,031 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:29,031 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/C, priority=13, startTime=1733732188544; duration=0sec 2024-12-09T08:16:29,031 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:29,031 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:C 2024-12-09T08:16:29,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T08:16:29,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:29,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:29,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:29,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732249181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:29,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732249191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:29,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/8ae6c82c260f4fa286e724b5fd283954 2024-12-09T08:16:29,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/80685c96fd964a59952de221b849122f is 50, key is test_row_0/B:col10/1733732188197/Put/seqid=0 2024-12-09T08:16:29,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742325_1501 (size=12301) 2024-12-09T08:16:29,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T08:16:29,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:29,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:29,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:29,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T08:16:29,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732249361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:29,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T08:16:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:29,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:29,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732249501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:29,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732249511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:29,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T08:16:29,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:29,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:29,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:29,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/80685c96fd964a59952de221b849122f 2024-12-09T08:16:29,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/af750702f4d8485e819e90192e218882 is 50, key is test_row_0/C:col10/1733732188197/Put/seqid=0 2024-12-09T08:16:29,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742326_1502 (size=12301) 2024-12-09T08:16:29,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T08:16:29,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:29,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:29,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:29,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T08:16:29,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T08:16:29,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:29,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:29,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:29,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:30,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732250012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:30,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732250031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:30,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/af750702f4d8485e819e90192e218882 2024-12-09T08:16:30,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/8ae6c82c260f4fa286e724b5fd283954 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8ae6c82c260f4fa286e724b5fd283954 2024-12-09T08:16:30,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8ae6c82c260f4fa286e724b5fd283954, entries=200, sequenceid=304, filesize=14.4 K 2024-12-09T08:16:30,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/80685c96fd964a59952de221b849122f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/80685c96fd964a59952de221b849122f 2024-12-09T08:16:30,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T08:16:30,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/80685c96fd964a59952de221b849122f, entries=150, sequenceid=304, filesize=12.0 K 2024-12-09T08:16:30,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:30,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:30,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:30,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:30,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:30,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/af750702f4d8485e819e90192e218882 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/af750702f4d8485e819e90192e218882 2024-12-09T08:16:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:30,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/af750702f4d8485e819e90192e218882, entries=150, sequenceid=304, filesize=12.0 K 2024-12-09T08:16:30,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 6b37acb92571896382ef9b774ba15dad in 1284ms, sequenceid=304, compaction requested=false 2024-12-09T08:16:30,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:30,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T08:16:30,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:30,215 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-09T08:16:30,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:30,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:30,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:30,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:30,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:30,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:30,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/03fe55a539d34e65a03d52ad818e9674 is 50, key is test_row_0/A:col10/1733732188813/Put/seqid=0 2024-12-09T08:16:30,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742327_1503 (size=12301) 2024-12-09T08:16:30,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:30,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732190458, max timestamp=1733732190486Num uniques 859; 2024-12-09T08:16:30,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732190458, max timestamp=1733732190486Num uniques 859; 2024-12-09T08:16:30,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:16:30,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:16:30,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732190458, max timestamp=1733732190486Num uniques 859; 2024-12-09T08:16:30,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:16:30,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733732190593, max timestamp=1733732190610Num uniques 1018; 2024-12-09T08:16:30,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733732190593, max timestamp=1733732190610Num uniques 1018; 2024-12-09T08:16:30,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T08:16:30,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733732190593, max timestamp=1733732190610Num uniques 1018; 2024-12-09T08:16:30,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T08:16:30,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T08:16:30,621 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/03fe55a539d34e65a03d52ad818e9674 2024-12-09T08:16:30,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/0dad0716efc840d9b14d86b53e665edf is 50, key is test_row_0/B:col10/1733732188813/Put/seqid=0 2024-12-09T08:16:30,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742328_1504 (size=12301) 2024-12-09T08:16:30,631 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/0dad0716efc840d9b14d86b53e665edf 2024-12-09T08:16:30,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/ac131ec555d5455d8e57f64842aeda12 is 50, key is test_row_0/C:col10/1733732188813/Put/seqid=0 2024-12-09T08:16:30,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742329_1505 (size=12301) 2024-12-09T08:16:30,646 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/ac131ec555d5455d8e57f64842aeda12 2024-12-09T08:16:30,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/03fe55a539d34e65a03d52ad818e9674 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/03fe55a539d34e65a03d52ad818e9674 2024-12-09T08:16:30,653 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/03fe55a539d34e65a03d52ad818e9674, entries=150, sequenceid=316, filesize=12.0 K 2024-12-09T08:16:30,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/0dad0716efc840d9b14d86b53e665edf as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/0dad0716efc840d9b14d86b53e665edf 2024-12-09T08:16:30,657 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/0dad0716efc840d9b14d86b53e665edf, entries=150, sequenceid=316, filesize=12.0 K 2024-12-09T08:16:30,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/ac131ec555d5455d8e57f64842aeda12 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ac131ec555d5455d8e57f64842aeda12 2024-12-09T08:16:30,660 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ac131ec555d5455d8e57f64842aeda12, entries=150, sequenceid=316, filesize=12.0 K 2024-12-09T08:16:30,661 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=73.80 KB/75570 for 6b37acb92571896382ef9b774ba15dad in 446ms, sequenceid=316, compaction requested=true 2024-12-09T08:16:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-09T08:16:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-09T08:16:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:30,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=80.51 KB heapSize=224.77 KB 2024-12-09T08:16:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:30,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-09T08:16:30,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9770 sec 2024-12-09T08:16:30,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9820 sec 2024-12-09T08:16:30,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/6972c9faa09d4c6eb011cf2bba565b69 is 50, key is test_row_0/A:col10/1733732190639/Put/seqid=0 2024-12-09T08:16:30,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742330_1506 (size=14741) 2024-12-09T08:16:30,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732250759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:30,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:30,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:30,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T08:16:30,831 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:30,831 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-09T08:16:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T08:16:30,833 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:30,833 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:30,833 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:30,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732250871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T08:16:30,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:30,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:30,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:30,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:30,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:30,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732251030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:31,037 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 6b37acb92571896382ef9b774ba15dad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:31,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:31,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:31,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:31,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732251051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:31,056 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 6b37acb92571896382ef9b774ba15dad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:31,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/6972c9faa09d4c6eb011cf2bba565b69 2024-12-09T08:16:31,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/f0a260d050e5459d8e70c6bf7d07c9ec is 50, key is test_row_0/B:col10/1733732190639/Put/seqid=0 2024-12-09T08:16:31,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742331_1507 (size=12301) 2024-12-09T08:16:31,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732251081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:31,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:31,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T08:16:31,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732251390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:31,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:31,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:31,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T08:16:31,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/f0a260d050e5459d8e70c6bf7d07c9ec 2024-12-09T08:16:31,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/219e88a305434a8080af786866ad7d96 is 50, key is test_row_0/C:col10/1733732190639/Put/seqid=0 2024-12-09T08:16:31,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742332_1508 (size=12301) 2024-12-09T08:16:31,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:31,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:31,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/219e88a305434a8080af786866ad7d96 2024-12-09T08:16:31,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/6972c9faa09d4c6eb011cf2bba565b69 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/6972c9faa09d4c6eb011cf2bba565b69 2024-12-09T08:16:31,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/6972c9faa09d4c6eb011cf2bba565b69, entries=200, sequenceid=362, filesize=14.4 K 2024-12-09T08:16:31,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/f0a260d050e5459d8e70c6bf7d07c9ec as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/f0a260d050e5459d8e70c6bf7d07c9ec 2024-12-09T08:16:31,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:31,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:31,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:31,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/f0a260d050e5459d8e70c6bf7d07c9ec, entries=150, sequenceid=362, filesize=12.0 K 2024-12-09T08:16:31,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/219e88a305434a8080af786866ad7d96 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/219e88a305434a8080af786866ad7d96 2024-12-09T08:16:31,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732251901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:31,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/219e88a305434a8080af786866ad7d96, entries=150, sequenceid=362, filesize=12.0 K 2024-12-09T08:16:31,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~224.72 KB/230112, currentSize=100.63 KB/103050 for 6b37acb92571896382ef9b774ba15dad in 1244ms, sequenceid=362, compaction requested=true 2024-12-09T08:16:31,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:31,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:31,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:31,907 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:16:31,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:31,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:31,907 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:16:31,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:31,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:31,908 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49836 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:16:31,908 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:16:31,908 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/A is initiating minor compaction (all files) 2024-12-09T08:16:31,908 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/B is initiating minor compaction (all files) 2024-12-09T08:16:31,908 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/B in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,908 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/A in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:31,908 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/a44e0c2bb205403d8660f05be82a5068, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/80685c96fd964a59952de221b849122f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/0dad0716efc840d9b14d86b53e665edf, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/f0a260d050e5459d8e70c6bf7d07c9ec] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=48.7 K 2024-12-09T08:16:31,908 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/1b35715f95f6428bb5c6b5eba1fe5aa1, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8ae6c82c260f4fa286e724b5fd283954, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/03fe55a539d34e65a03d52ad818e9674, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/6972c9faa09d4c6eb011cf2bba565b69] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=53.4 K 2024-12-09T08:16:31,908 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a44e0c2bb205403d8660f05be82a5068, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733732185919 2024-12-09T08:16:31,908 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b35715f95f6428bb5c6b5eba1fe5aa1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733732185919 2024-12-09T08:16:31,908 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 80685c96fd964a59952de221b849122f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733732188189 2024-12-09T08:16:31,908 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ae6c82c260f4fa286e724b5fd283954, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733732188163 2024-12-09T08:16:31,908 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dad0716efc840d9b14d86b53e665edf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733732188813 2024-12-09T08:16:31,909 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03fe55a539d34e65a03d52ad818e9674, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733732188813 2024-12-09T08:16:31,909 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting f0a260d050e5459d8e70c6bf7d07c9ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733732190639 2024-12-09T08:16:31,909 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6972c9faa09d4c6eb011cf2bba565b69, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733732190639 2024-12-09T08:16:31,916 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:31,916 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/d30aa9b5f19d4c3fa91392a577e5c9b2 is 50, key is test_row_0/B:col10/1733732190639/Put/seqid=0 2024-12-09T08:16:31,917 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#A#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:31,918 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/e184c2bd2921460d8e6ebf92735a05be is 50, key is test_row_0/A:col10/1733732190639/Put/seqid=0 2024-12-09T08:16:31,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742333_1509 (size=13119) 2024-12-09T08:16:31,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742334_1510 (size=13119) 2024-12-09T08:16:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T08:16:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:32,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T08:16:32,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:32,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:32,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:32,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:32,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:32,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:32,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/2a65dc4c4519459f949e03b374ff45ab is 50, key is test_row_0/A:col10/1733732192006/Put/seqid=0 2024-12-09T08:16:32,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742335_1511 (size=14741) 2024-12-09T08:16:32,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/2a65dc4c4519459f949e03b374ff45ab 2024-12-09T08:16:32,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/e5a190ac9e46410cb885b578b1ea1ee5 is 50, key is test_row_0/B:col10/1733732192006/Put/seqid=0 2024-12-09T08:16:32,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742336_1512 (size=12301) 2024-12-09T08:16:32,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:32,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:32,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:32,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732252089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732252089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:32,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:32,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:32,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732252211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:32,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732252211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:32,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:32,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:32,325 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/d30aa9b5f19d4c3fa91392a577e5c9b2 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/d30aa9b5f19d4c3fa91392a577e5c9b2 2024-12-09T08:16:32,329 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/e184c2bd2921460d8e6ebf92735a05be as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/e184c2bd2921460d8e6ebf92735a05be 2024-12-09T08:16:32,329 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/B of 6b37acb92571896382ef9b774ba15dad into d30aa9b5f19d4c3fa91392a577e5c9b2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:32,329 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:32,329 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/B, priority=12, startTime=1733732191907; duration=0sec 2024-12-09T08:16:32,329 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:32,329 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:B 2024-12-09T08:16:32,329 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:16:32,330 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49836 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T08:16:32,330 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/C is initiating minor compaction (all files) 2024-12-09T08:16:32,330 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/C in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,330 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/a21c0c3357a143719aa32053a6b25f13, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/af750702f4d8485e819e90192e218882, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ac131ec555d5455d8e57f64842aeda12, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/219e88a305434a8080af786866ad7d96] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=48.7 K 2024-12-09T08:16:32,331 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a21c0c3357a143719aa32053a6b25f13, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733732185919 2024-12-09T08:16:32,331 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting af750702f4d8485e819e90192e218882, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733732188189 2024-12-09T08:16:32,331 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ac131ec555d5455d8e57f64842aeda12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733732188813 2024-12-09T08:16:32,331 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 219e88a305434a8080af786866ad7d96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733732190639 2024-12-09T08:16:32,333 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/A of 6b37acb92571896382ef9b774ba15dad into e184c2bd2921460d8e6ebf92735a05be(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:32,333 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:32,333 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/A, priority=12, startTime=1733732191907; duration=0sec 2024-12-09T08:16:32,333 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:32,333 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:A 2024-12-09T08:16:32,337 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#C#compaction#441 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:32,337 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/d2634520fa824b2c8cff217ee22e9c86 is 50, key is test_row_0/C:col10/1733732190639/Put/seqid=0 2024-12-09T08:16:32,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742337_1513 (size=13119) 2024-12-09T08:16:32,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:32,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:32,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732252421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:32,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/e5a190ac9e46410cb885b578b1ea1ee5 2024-12-09T08:16:32,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/669658180efb4aa2a5e8e06cd9b91dde is 50, key is test_row_0/C:col10/1733732192006/Put/seqid=0 2024-12-09T08:16:32,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732252431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:32,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742338_1514 (size=12301) 2024-12-09T08:16:32,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:32,511 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T08:16:32,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:32,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:32,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:32,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732252730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:32,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732252741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:32,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:32,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:32,745 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/d2634520fa824b2c8cff217ee22e9c86 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/d2634520fa824b2c8cff217ee22e9c86 2024-12-09T08:16:32,748 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/C of 6b37acb92571896382ef9b774ba15dad into d2634520fa824b2c8cff217ee22e9c86(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:32,748 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:32,748 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/C, priority=12, startTime=1733732191907; duration=0sec 2024-12-09T08:16:32,748 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:32,748 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:C 2024-12-09T08:16:32,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:32,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:32,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/669658180efb4aa2a5e8e06cd9b91dde 2024-12-09T08:16:32,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/2a65dc4c4519459f949e03b374ff45ab as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/2a65dc4c4519459f949e03b374ff45ab 2024-12-09T08:16:32,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/2a65dc4c4519459f949e03b374ff45ab, entries=200, sequenceid=381, filesize=14.4 K 2024-12-09T08:16:32,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/e5a190ac9e46410cb885b578b1ea1ee5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/e5a190ac9e46410cb885b578b1ea1ee5 2024-12-09T08:16:32,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/e5a190ac9e46410cb885b578b1ea1ee5, entries=150, sequenceid=381, filesize=12.0 K 2024-12-09T08:16:32,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/669658180efb4aa2a5e8e06cd9b91dde as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/669658180efb4aa2a5e8e06cd9b91dde 2024-12-09T08:16:32,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/669658180efb4aa2a5e8e06cd9b91dde, entries=150, sequenceid=381, filesize=12.0 K 2024-12-09T08:16:32,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 6b37acb92571896382ef9b774ba15dad in 843ms, sequenceid=381, compaction requested=false 2024-12-09T08:16:32,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:32,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T08:16:32,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:32,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:32,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:32,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:32,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:32,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:32,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/da6cbbf1ddb34dd192566c081516d6c1 is 50, key is test_row_0/A:col10/1733732192045/Put/seqid=0 2024-12-09T08:16:32,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742339_1515 (size=14741) 2024-12-09T08:16:32,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:32,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T08:16:33,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732253014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:33,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1733732253061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:33,067 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 6b37acb92571896382ef9b774ba15dad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:33,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1733732253071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:33,077 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 6b37acb92571896382ef9b774ba15dad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:33,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732253131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:33,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:33,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46988 deadline: 1733732253241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:33,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:33,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47030 deadline: 1733732253251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:33,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:33,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:33,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/da6cbbf1ddb34dd192566c081516d6c1 2024-12-09T08:16:33,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/4a7c0781842640a19c0a411e7b36d570 is 50, key is test_row_0/B:col10/1733732192045/Put/seqid=0 2024-12-09T08:16:33,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742340_1516 (size=12301) 2024-12-09T08:16:33,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732253341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:33,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/4a7c0781842640a19c0a411e7b36d570 2024-12-09T08:16:33,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/c37422611ce948f1bff542023714f320 is 50, key is test_row_0/C:col10/1733732192045/Put/seqid=0 2024-12-09T08:16:33,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742341_1517 (size=12301) 2024-12-09T08:16:33,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:33,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:33,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:33,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,650 INFO [Thread-1973 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1973. 2024-12-09T08:16:33,650 DEBUG [Thread-1973 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:33,650 DEBUG [Thread-1973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:33,650 DEBUG [Thread-1973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:33,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:33,651 INFO [Thread-1981 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1981. 2024-12-09T08:16:33,651 DEBUG [Thread-1981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:33,651 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:33,651 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:33,651 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:33,652 INFO [Thread-1979 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1979. 2024-12-09T08:16:33,653 DEBUG [Thread-1979 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:33,653 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:33,653 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:33,653 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:33,657 INFO [Thread-1975 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1975. 2024-12-09T08:16:33,657 DEBUG [Thread-1975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:33,657 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:33,657 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:33,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:33,658 INFO [Thread-1977 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1977. 2024-12-09T08:16:33,658 DEBUG [Thread-1977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:33,658 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:33,658 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:33,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:33,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46986 deadline: 1733732253661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 , the old value is region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., hostname=5bda946059d1,43255,1733732044053, seqNum=2 because the exception is null or not the one we care about 2024-12-09T08:16:33,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:33,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/c37422611ce948f1bff542023714f320 2024-12-09T08:16:33,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/da6cbbf1ddb34dd192566c081516d6c1 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/da6cbbf1ddb34dd192566c081516d6c1 2024-12-09T08:16:33,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/da6cbbf1ddb34dd192566c081516d6c1, entries=200, sequenceid=399, filesize=14.4 K 2024-12-09T08:16:33,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/4a7c0781842640a19c0a411e7b36d570 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/4a7c0781842640a19c0a411e7b36d570 2024-12-09T08:16:33,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/4a7c0781842640a19c0a411e7b36d570, entries=150, sequenceid=399, filesize=12.0 K 2024-12-09T08:16:33,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/c37422611ce948f1bff542023714f320 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/c37422611ce948f1bff542023714f320 2024-12-09T08:16:33,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/c37422611ce948f1bff542023714f320, entries=150, sequenceid=399, filesize=12.0 K 2024-12-09T08:16:33,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6b37acb92571896382ef9b774ba15dad in 865ms, sequenceid=399, compaction requested=true 2024-12-09T08:16:33,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:33,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:33,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:33,793 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:33,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:33,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:33,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b37acb92571896382ef9b774ba15dad:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:33,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:33,793 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:33,794 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:33,794 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:33,794 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/B is initiating minor compaction (all files) 2024-12-09T08:16:33,794 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/A is initiating minor compaction (all files) 2024-12-09T08:16:33,794 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/B in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,794 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/A in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,794 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/d30aa9b5f19d4c3fa91392a577e5c9b2, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/e5a190ac9e46410cb885b578b1ea1ee5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/4a7c0781842640a19c0a411e7b36d570] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=36.8 K 2024-12-09T08:16:33,794 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/e184c2bd2921460d8e6ebf92735a05be, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/2a65dc4c4519459f949e03b374ff45ab, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/da6cbbf1ddb34dd192566c081516d6c1] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=41.6 K 2024-12-09T08:16:33,795 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting d30aa9b5f19d4c3fa91392a577e5c9b2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733732190639 2024-12-09T08:16:33,795 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting e184c2bd2921460d8e6ebf92735a05be, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733732190639 2024-12-09T08:16:33,795 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting e5a190ac9e46410cb885b578b1ea1ee5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733732190744 2024-12-09T08:16:33,795 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a65dc4c4519459f949e03b374ff45ab, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733732190667 2024-12-09T08:16:33,795 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a7c0781842640a19c0a411e7b36d570, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733732192045 2024-12-09T08:16:33,795 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting da6cbbf1ddb34dd192566c081516d6c1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733732192045 2024-12-09T08:16:33,800 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#B#compaction#446 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:33,800 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#A#compaction#447 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:33,801 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/143c10341e7a47748d47939979521832 is 50, key is test_row_0/A:col10/1733732192045/Put/seqid=0 2024-12-09T08:16:33,801 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/eab245f6f54e4bee823946fb39e5ecfe is 50, key is test_row_0/B:col10/1733732192045/Put/seqid=0 2024-12-09T08:16:33,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742343_1519 (size=13221) 2024-12-09T08:16:33,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742342_1518 (size=13221) 2024-12-09T08:16:33,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T08:16:33,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:33,889 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T08:16:33,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:33,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:33,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:33,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:33,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:33,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:33,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/8c3cc0a5f37e49e093ce99f0ccaa28e6 is 50, key is test_row_0/A:col10/1733732193005/Put/seqid=0 2024-12-09T08:16:33,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742344_1520 (size=12301) 2024-12-09T08:16:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:34,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. as already flushing 2024-12-09T08:16:34,173 INFO [Thread-1968 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1968. 2024-12-09T08:16:34,173 DEBUG [Thread-1968 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:34,173 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:34,173 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:34,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:34,208 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/143c10341e7a47748d47939979521832 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/143c10341e7a47748d47939979521832 2024-12-09T08:16:34,208 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/eab245f6f54e4bee823946fb39e5ecfe as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/eab245f6f54e4bee823946fb39e5ecfe 2024-12-09T08:16:34,212 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/B of 6b37acb92571896382ef9b774ba15dad into eab245f6f54e4bee823946fb39e5ecfe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:34,212 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/A of 6b37acb92571896382ef9b774ba15dad into 143c10341e7a47748d47939979521832(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:34,212 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:34,212 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:34,212 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/B, priority=13, startTime=1733732193793; duration=0sec 2024-12-09T08:16:34,212 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/A, priority=13, startTime=1733732193793; duration=0sec 2024-12-09T08:16:34,212 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:34,212 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:B 2024-12-09T08:16:34,212 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:34,212 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:A 2024-12-09T08:16:34,212 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:34,213 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:34,213 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 6b37acb92571896382ef9b774ba15dad/C is initiating minor compaction (all files) 2024-12-09T08:16:34,213 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b37acb92571896382ef9b774ba15dad/C in TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:34,213 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/d2634520fa824b2c8cff217ee22e9c86, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/669658180efb4aa2a5e8e06cd9b91dde, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/c37422611ce948f1bff542023714f320] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp, totalSize=36.8 K 2024-12-09T08:16:34,213 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting d2634520fa824b2c8cff217ee22e9c86, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733732190639 2024-12-09T08:16:34,213 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 669658180efb4aa2a5e8e06cd9b91dde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733732190744 2024-12-09T08:16:34,214 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting c37422611ce948f1bff542023714f320, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733732192045 2024-12-09T08:16:34,219 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b37acb92571896382ef9b774ba15dad#C#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:34,219 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/d3da9f7ee0b842dab248e2e8b8b44ca3 is 50, key is test_row_0/C:col10/1733732192045/Put/seqid=0 2024-12-09T08:16:34,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742345_1521 (size=13221) 2024-12-09T08:16:34,263 INFO [Thread-1966 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1966. 2024-12-09T08:16:34,263 INFO [Thread-1970 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1970. 2024-12-09T08:16:34,263 DEBUG [Thread-1966 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:34,263 DEBUG [Thread-1970 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:34,263 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:34,263 DEBUG [Thread-1970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:34,263 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:34,263 DEBUG [Thread-1970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:34,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:34,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:34,302 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/8c3cc0a5f37e49e093ce99f0ccaa28e6 2024-12-09T08:16:34,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/3b30614439664c699e37f7de4f6672d7 is 50, key is test_row_0/B:col10/1733732193005/Put/seqid=0 2024-12-09T08:16:34,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742346_1522 (size=12301) 2024-12-09T08:16:34,629 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/d3da9f7ee0b842dab248e2e8b8b44ca3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/d3da9f7ee0b842dab248e2e8b8b44ca3 2024-12-09T08:16:34,636 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b37acb92571896382ef9b774ba15dad/C of 6b37acb92571896382ef9b774ba15dad into d3da9f7ee0b842dab248e2e8b8b44ca3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:34,636 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:34,636 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., storeName=6b37acb92571896382ef9b774ba15dad/C, priority=13, startTime=1733732193793; duration=0sec 2024-12-09T08:16:34,636 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:34,636 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b37acb92571896382ef9b774ba15dad:C 2024-12-09T08:16:34,711 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/3b30614439664c699e37f7de4f6672d7 2024-12-09T08:16:34,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/0e7b231af49a4a219badbd7b598a9186 is 50, key is test_row_0/C:col10/1733732193005/Put/seqid=0 2024-12-09T08:16:34,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742347_1523 (size=12301) 2024-12-09T08:16:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T08:16:35,124 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/0e7b231af49a4a219badbd7b598a9186 2024-12-09T08:16:35,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/8c3cc0a5f37e49e093ce99f0ccaa28e6 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8c3cc0a5f37e49e093ce99f0ccaa28e6 2024-12-09T08:16:35,133 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8c3cc0a5f37e49e093ce99f0ccaa28e6, entries=150, sequenceid=416, filesize=12.0 K 2024-12-09T08:16:35,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/3b30614439664c699e37f7de4f6672d7 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/3b30614439664c699e37f7de4f6672d7 2024-12-09T08:16:35,136 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/3b30614439664c699e37f7de4f6672d7, entries=150, sequenceid=416, filesize=12.0 K 2024-12-09T08:16:35,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/0e7b231af49a4a219badbd7b598a9186 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/0e7b231af49a4a219badbd7b598a9186 2024-12-09T08:16:35,140 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/0e7b231af49a4a219badbd7b598a9186, entries=150, sequenceid=416, filesize=12.0 K 2024-12-09T08:16:35,141 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for 6b37acb92571896382ef9b774ba15dad in 1252ms, sequenceid=416, compaction requested=false 2024-12-09T08:16:35,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 6b37acb92571896382ef9b774ba15dad: 2024-12-09T08:16:35,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:35,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-09T08:16:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-09T08:16:35,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-09T08:16:35,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.3090 sec 2024-12-09T08:16:35,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 4.3140 sec 2024-12-09T08:16:37,073 INFO [Thread-1962 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1962. 2024-12-09T08:16:37,073 DEBUG [Thread-1962 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:37,073 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:37,073 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:37,073 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:37,103 INFO [Thread-1964 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1964. 2024-12-09T08:16:37,103 DEBUG [Thread-1964 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:16:37,103 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:37,103 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:37,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T08:16:39,013 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:39,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T08:16:39,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-12-09T08:16:39,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-12-09T08:16:39,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-09T08:16:39,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 159 2024-12-09T08:16:39,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-09T08:16:39,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T08:16:39,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T08:16:39,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1542 2024-12-09T08:16:39,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4626 rows 2024-12-09T08:16:39,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1561 2024-12-09T08:16:39,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4680 rows 2024-12-09T08:16:39,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1548 2024-12-09T08:16:39,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4644 rows 2024-12-09T08:16:39,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1540 2024-12-09T08:16:39,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4620 rows 2024-12-09T08:16:39,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1549 2024-12-09T08:16:39,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4644 rows 2024-12-09T08:16:39,014 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T08:16:39,014 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:16:39,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:39,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:39,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:16:39,017 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T08:16:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T08:16:39,023 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732199023"}]},"ts":"1733732199023"} 2024-12-09T08:16:39,025 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T08:16:39,025 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T08:16:39,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T08:16:39,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b37acb92571896382ef9b774ba15dad, UNASSIGN}] 2024-12-09T08:16:39,027 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b37acb92571896382ef9b774ba15dad, UNASSIGN 2024-12-09T08:16:39,028 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=6b37acb92571896382ef9b774ba15dad, regionState=CLOSING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:16:39,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b37acb92571896382ef9b774ba15dad, UNASSIGN because future has completed 2024-12-09T08:16:39,029 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:16:39,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:16:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T08:16:39,181 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:39,181 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:16:39,181 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 6b37acb92571896382ef9b774ba15dad, disabling compactions & flushes 2024-12-09T08:16:39,182 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:39,182 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:39,182 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. after waiting 0 ms 2024-12-09T08:16:39,182 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:39,182 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 6b37acb92571896382ef9b774ba15dad 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T08:16:39,182 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=A 2024-12-09T08:16:39,182 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:39,182 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=B 2024-12-09T08:16:39,182 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:39,182 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b37acb92571896382ef9b774ba15dad, store=C 2024-12-09T08:16:39,182 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:39,186 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/28d57ef803964eda91570a59d0abfcb1 is 50, key is test_row_0/A:col10/1733732194262/Put/seqid=0 2024-12-09T08:16:39,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742348_1524 (size=12301) 2024-12-09T08:16:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T08:16:39,592 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/28d57ef803964eda91570a59d0abfcb1 2024-12-09T08:16:39,604 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/bbb115cebddd42eab35b339a76c6ad7a is 50, key is test_row_0/B:col10/1733732194262/Put/seqid=0 2024-12-09T08:16:39,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742349_1525 (size=12301) 2024-12-09T08:16:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T08:16:40,009 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/bbb115cebddd42eab35b339a76c6ad7a 2024-12-09T08:16:40,023 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/21df66695d664f47aa4b5465eefd813a is 50, key is test_row_0/C:col10/1733732194262/Put/seqid=0 2024-12-09T08:16:40,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742350_1526 (size=12301) 2024-12-09T08:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T08:16:40,428 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/21df66695d664f47aa4b5465eefd813a 2024-12-09T08:16:40,438 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/A/28d57ef803964eda91570a59d0abfcb1 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/28d57ef803964eda91570a59d0abfcb1 2024-12-09T08:16:40,445 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/28d57ef803964eda91570a59d0abfcb1, entries=150, sequenceid=427, filesize=12.0 K 2024-12-09T08:16:40,446 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/B/bbb115cebddd42eab35b339a76c6ad7a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/bbb115cebddd42eab35b339a76c6ad7a 2024-12-09T08:16:40,450 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/bbb115cebddd42eab35b339a76c6ad7a, entries=150, sequenceid=427, filesize=12.0 K 2024-12-09T08:16:40,451 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/.tmp/C/21df66695d664f47aa4b5465eefd813a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/21df66695d664f47aa4b5465eefd813a 2024-12-09T08:16:40,455 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/21df66695d664f47aa4b5465eefd813a, entries=150, sequenceid=427, filesize=12.0 K 2024-12-09T08:16:40,456 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6b37acb92571896382ef9b774ba15dad in 1274ms, sequenceid=427, compaction requested=true 2024-12-09T08:16:40,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/a9df4b01bb2c4c45ba9777ea9e625e94, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/4f86c2a83aed4879bcb79f0a3218c591, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/e4320f829596485e8a4d493f3390c3ea, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/7ea2e25e868c4578a5f5d3f700a4969e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/722c5602091f4d08b346cc037927c585, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/0ce71a89c7ff409096cac7b345fc72ec, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/d3ae8216ec83442a8851b82eba8e44f7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/bf5d451504fb4d7f89e15b8cc8001935, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/887b1d93f4f94ed88d50a2433ca0e7c5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8ad983a64acf4df18a441fd9c12d723f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/bb2a2a7a45a24b0da0f559f3055365b4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/890227c4ab734459bf8e3dd90fc4eb72, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/7e677ef92aa9495d988e8744b4f27bff, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/ba1cc4774a3b47d0a78ef06c177260d6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/32c2a12fe88b4e8fb52f98d77877ebea, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/de7ae6c078f84d7ab8687df4d888d2b4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/88e1cb948f1a4d33a84ffbb0ae4243de, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/b51cf2c3f07f466182666cf47bb7da31, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/f4e1562ea7964d09905b225bef1499de, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/1b35715f95f6428bb5c6b5eba1fe5aa1, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/5e5eec2894ca40e4ba1a370ff9d940c1, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8ae6c82c260f4fa286e724b5fd283954, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/03fe55a539d34e65a03d52ad818e9674, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/6972c9faa09d4c6eb011cf2bba565b69, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/e184c2bd2921460d8e6ebf92735a05be, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/2a65dc4c4519459f949e03b374ff45ab, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/da6cbbf1ddb34dd192566c081516d6c1] to archive 2024-12-09T08:16:40,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:16:40,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/a9df4b01bb2c4c45ba9777ea9e625e94 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/a9df4b01bb2c4c45ba9777ea9e625e94 2024-12-09T08:16:40,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/4f86c2a83aed4879bcb79f0a3218c591 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/4f86c2a83aed4879bcb79f0a3218c591 2024-12-09T08:16:40,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/e4320f829596485e8a4d493f3390c3ea to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/e4320f829596485e8a4d493f3390c3ea 2024-12-09T08:16:40,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/7ea2e25e868c4578a5f5d3f700a4969e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/7ea2e25e868c4578a5f5d3f700a4969e 2024-12-09T08:16:40,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/722c5602091f4d08b346cc037927c585 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/722c5602091f4d08b346cc037927c585 2024-12-09T08:16:40,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/0ce71a89c7ff409096cac7b345fc72ec to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/0ce71a89c7ff409096cac7b345fc72ec 2024-12-09T08:16:40,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/d3ae8216ec83442a8851b82eba8e44f7 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/d3ae8216ec83442a8851b82eba8e44f7 2024-12-09T08:16:40,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/bf5d451504fb4d7f89e15b8cc8001935 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/bf5d451504fb4d7f89e15b8cc8001935 2024-12-09T08:16:40,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/887b1d93f4f94ed88d50a2433ca0e7c5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/887b1d93f4f94ed88d50a2433ca0e7c5 2024-12-09T08:16:40,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8ad983a64acf4df18a441fd9c12d723f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8ad983a64acf4df18a441fd9c12d723f 2024-12-09T08:16:40,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/bb2a2a7a45a24b0da0f559f3055365b4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/bb2a2a7a45a24b0da0f559f3055365b4 2024-12-09T08:16:40,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/890227c4ab734459bf8e3dd90fc4eb72 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/890227c4ab734459bf8e3dd90fc4eb72 2024-12-09T08:16:40,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/7e677ef92aa9495d988e8744b4f27bff to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/7e677ef92aa9495d988e8744b4f27bff 2024-12-09T08:16:40,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/ba1cc4774a3b47d0a78ef06c177260d6 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/ba1cc4774a3b47d0a78ef06c177260d6 2024-12-09T08:16:40,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/32c2a12fe88b4e8fb52f98d77877ebea to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/32c2a12fe88b4e8fb52f98d77877ebea 2024-12-09T08:16:40,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/de7ae6c078f84d7ab8687df4d888d2b4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/de7ae6c078f84d7ab8687df4d888d2b4 2024-12-09T08:16:40,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/88e1cb948f1a4d33a84ffbb0ae4243de to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/88e1cb948f1a4d33a84ffbb0ae4243de 2024-12-09T08:16:40,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/b51cf2c3f07f466182666cf47bb7da31 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/b51cf2c3f07f466182666cf47bb7da31 2024-12-09T08:16:40,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/f4e1562ea7964d09905b225bef1499de to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/f4e1562ea7964d09905b225bef1499de 2024-12-09T08:16:40,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/1b35715f95f6428bb5c6b5eba1fe5aa1 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/1b35715f95f6428bb5c6b5eba1fe5aa1 2024-12-09T08:16:40,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/5e5eec2894ca40e4ba1a370ff9d940c1 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/5e5eec2894ca40e4ba1a370ff9d940c1 2024-12-09T08:16:40,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8ae6c82c260f4fa286e724b5fd283954 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8ae6c82c260f4fa286e724b5fd283954 2024-12-09T08:16:40,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/03fe55a539d34e65a03d52ad818e9674 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/03fe55a539d34e65a03d52ad818e9674 2024-12-09T08:16:40,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/6972c9faa09d4c6eb011cf2bba565b69 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/6972c9faa09d4c6eb011cf2bba565b69 2024-12-09T08:16:40,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/e184c2bd2921460d8e6ebf92735a05be to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/e184c2bd2921460d8e6ebf92735a05be 2024-12-09T08:16:40,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/2a65dc4c4519459f949e03b374ff45ab to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/2a65dc4c4519459f949e03b374ff45ab 2024-12-09T08:16:40,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/da6cbbf1ddb34dd192566c081516d6c1 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/da6cbbf1ddb34dd192566c081516d6c1 2024-12-09T08:16:40,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2e2c7e53b65b444080402acd97dc62fb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/c9e63a69e7db4243b6dd9d503741c5c7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/24bd5dc35dd74e21a2de0cff6ef3acf6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/fb5b3254305f426798057942aa548198, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/e40268c04efa47edb5a0d7a73f4fe67a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/1d3ba3fd3b96417a8ac96361ba767c2c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/43d88e0d508945ffb73a648f5f0f3b86, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/6a4e6e05b044459a8a4fece94eb9bbf7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/18653e23f962412897fb82ab8dc14c2c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/a8acc6cd5fe249f7902694d75eb7658b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/03c7aa4ad85d46fca89f7be4a9605320, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2f2361340cde46f5aa856bc47594700c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/9820b7857ffc4e978297765724e17fe6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/8fa46eba8894430883d9f2e01ca8509b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/9f5877d375cc402c89a5332c83b2b730, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/7120c3e4d2fa40f6a7fe8c016583d73c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/c3dacf4284d9422c914d1ab39f3346cd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/cdd82597b78c4e9cb9378b8e0178efe6, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2481e1c30397498aa5bdd5ce5834e34a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/a44e0c2bb205403d8660f05be82a5068, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/3020f73b7749452ea5a0e54494e4b5ff, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/80685c96fd964a59952de221b849122f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/0dad0716efc840d9b14d86b53e665edf, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/d30aa9b5f19d4c3fa91392a577e5c9b2, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/f0a260d050e5459d8e70c6bf7d07c9ec, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/e5a190ac9e46410cb885b578b1ea1ee5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/4a7c0781842640a19c0a411e7b36d570] to archive 2024-12-09T08:16:40,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:16:40,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2e2c7e53b65b444080402acd97dc62fb to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2e2c7e53b65b444080402acd97dc62fb 2024-12-09T08:16:40,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/c9e63a69e7db4243b6dd9d503741c5c7 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/c9e63a69e7db4243b6dd9d503741c5c7 2024-12-09T08:16:40,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/24bd5dc35dd74e21a2de0cff6ef3acf6 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/24bd5dc35dd74e21a2de0cff6ef3acf6 2024-12-09T08:16:40,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/fb5b3254305f426798057942aa548198 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/fb5b3254305f426798057942aa548198 2024-12-09T08:16:40,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/e40268c04efa47edb5a0d7a73f4fe67a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/e40268c04efa47edb5a0d7a73f4fe67a 2024-12-09T08:16:40,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/1d3ba3fd3b96417a8ac96361ba767c2c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/1d3ba3fd3b96417a8ac96361ba767c2c 2024-12-09T08:16:40,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/43d88e0d508945ffb73a648f5f0f3b86 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/43d88e0d508945ffb73a648f5f0f3b86 2024-12-09T08:16:40,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/6a4e6e05b044459a8a4fece94eb9bbf7 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/6a4e6e05b044459a8a4fece94eb9bbf7 2024-12-09T08:16:40,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/18653e23f962412897fb82ab8dc14c2c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/18653e23f962412897fb82ab8dc14c2c 2024-12-09T08:16:40,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/a8acc6cd5fe249f7902694d75eb7658b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/a8acc6cd5fe249f7902694d75eb7658b 2024-12-09T08:16:40,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/03c7aa4ad85d46fca89f7be4a9605320 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/03c7aa4ad85d46fca89f7be4a9605320 2024-12-09T08:16:40,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2f2361340cde46f5aa856bc47594700c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2f2361340cde46f5aa856bc47594700c 2024-12-09T08:16:40,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/9820b7857ffc4e978297765724e17fe6 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/9820b7857ffc4e978297765724e17fe6 2024-12-09T08:16:40,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/8fa46eba8894430883d9f2e01ca8509b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/8fa46eba8894430883d9f2e01ca8509b 2024-12-09T08:16:40,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/9f5877d375cc402c89a5332c83b2b730 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/9f5877d375cc402c89a5332c83b2b730 2024-12-09T08:16:40,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/7120c3e4d2fa40f6a7fe8c016583d73c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/7120c3e4d2fa40f6a7fe8c016583d73c 2024-12-09T08:16:40,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/c3dacf4284d9422c914d1ab39f3346cd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/c3dacf4284d9422c914d1ab39f3346cd 2024-12-09T08:16:40,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/cdd82597b78c4e9cb9378b8e0178efe6 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/cdd82597b78c4e9cb9378b8e0178efe6 2024-12-09T08:16:40,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2481e1c30397498aa5bdd5ce5834e34a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/2481e1c30397498aa5bdd5ce5834e34a 2024-12-09T08:16:40,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/a44e0c2bb205403d8660f05be82a5068 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/a44e0c2bb205403d8660f05be82a5068 2024-12-09T08:16:40,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/3020f73b7749452ea5a0e54494e4b5ff to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/3020f73b7749452ea5a0e54494e4b5ff 2024-12-09T08:16:40,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/80685c96fd964a59952de221b849122f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/80685c96fd964a59952de221b849122f 2024-12-09T08:16:40,507 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/0dad0716efc840d9b14d86b53e665edf to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/0dad0716efc840d9b14d86b53e665edf 2024-12-09T08:16:40,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/d30aa9b5f19d4c3fa91392a577e5c9b2 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/d30aa9b5f19d4c3fa91392a577e5c9b2 2024-12-09T08:16:40,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/f0a260d050e5459d8e70c6bf7d07c9ec to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/f0a260d050e5459d8e70c6bf7d07c9ec 2024-12-09T08:16:40,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/e5a190ac9e46410cb885b578b1ea1ee5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/e5a190ac9e46410cb885b578b1ea1ee5 2024-12-09T08:16:40,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/4a7c0781842640a19c0a411e7b36d570 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/4a7c0781842640a19c0a411e7b36d570 2024-12-09T08:16:40,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/a70b67aa4bc9479c976dd199374a134c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/965a00fc0f104efcae1d15c325186c77, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/50e3d35865ee44c4872449957fa05eac, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/bd6d0546bd0f4871b60bcadd62a05df3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/710b1e44de3c4692a51f489c1a98038d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/15b371b5cb884329a15e24798d634b35, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/73ddf6b58d0443bfb3b1fb8057751d9a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/4f51225fe0af4efba80d3aa148bfebcd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/cf70fc2eacfd4ecf94a0cb1eb30a5844, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/15a994e13aaf4d50af567db1bef72efc, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ef087876310d4580b72a8c3a0e02432c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/b9ace48ca7d6488ba59bf7ff676c92db, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/f21627a57653436281a11171075a17ef, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/3f35cad63b854b77af8b064a492fa845, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ecc149855e3341d5ae8be45b00efa927, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/1e70114aba5b448cac57cce603da9999, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/4e92be2570974c699a72cfcb9e703b7c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/c0edb4315d3b4ac497c5fc2a73955a50, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/82ae508119f24f35a87627fd9d8d4dcb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/a21c0c3357a143719aa32053a6b25f13, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/252721d79e294c0eba752af9bf0fd649, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/af750702f4d8485e819e90192e218882, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ac131ec555d5455d8e57f64842aeda12, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/d2634520fa824b2c8cff217ee22e9c86, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/219e88a305434a8080af786866ad7d96, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/669658180efb4aa2a5e8e06cd9b91dde, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/c37422611ce948f1bff542023714f320] to archive 2024-12-09T08:16:40,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:16:40,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/a70b67aa4bc9479c976dd199374a134c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/a70b67aa4bc9479c976dd199374a134c 2024-12-09T08:16:40,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/965a00fc0f104efcae1d15c325186c77 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/965a00fc0f104efcae1d15c325186c77 2024-12-09T08:16:40,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/50e3d35865ee44c4872449957fa05eac to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/50e3d35865ee44c4872449957fa05eac 2024-12-09T08:16:40,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/bd6d0546bd0f4871b60bcadd62a05df3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/bd6d0546bd0f4871b60bcadd62a05df3 2024-12-09T08:16:40,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/710b1e44de3c4692a51f489c1a98038d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/710b1e44de3c4692a51f489c1a98038d 2024-12-09T08:16:40,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/15b371b5cb884329a15e24798d634b35 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/15b371b5cb884329a15e24798d634b35 2024-12-09T08:16:40,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/73ddf6b58d0443bfb3b1fb8057751d9a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/73ddf6b58d0443bfb3b1fb8057751d9a 2024-12-09T08:16:40,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/4f51225fe0af4efba80d3aa148bfebcd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/4f51225fe0af4efba80d3aa148bfebcd 2024-12-09T08:16:40,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/cf70fc2eacfd4ecf94a0cb1eb30a5844 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/cf70fc2eacfd4ecf94a0cb1eb30a5844 2024-12-09T08:16:40,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/15a994e13aaf4d50af567db1bef72efc to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/15a994e13aaf4d50af567db1bef72efc 2024-12-09T08:16:40,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ef087876310d4580b72a8c3a0e02432c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ef087876310d4580b72a8c3a0e02432c 2024-12-09T08:16:40,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/b9ace48ca7d6488ba59bf7ff676c92db to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/b9ace48ca7d6488ba59bf7ff676c92db 2024-12-09T08:16:40,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/f21627a57653436281a11171075a17ef to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/f21627a57653436281a11171075a17ef 2024-12-09T08:16:40,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/3f35cad63b854b77af8b064a492fa845 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/3f35cad63b854b77af8b064a492fa845 2024-12-09T08:16:40,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ecc149855e3341d5ae8be45b00efa927 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ecc149855e3341d5ae8be45b00efa927 2024-12-09T08:16:40,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/1e70114aba5b448cac57cce603da9999 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/1e70114aba5b448cac57cce603da9999 2024-12-09T08:16:40,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/4e92be2570974c699a72cfcb9e703b7c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/4e92be2570974c699a72cfcb9e703b7c 2024-12-09T08:16:40,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/c0edb4315d3b4ac497c5fc2a73955a50 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/c0edb4315d3b4ac497c5fc2a73955a50 2024-12-09T08:16:40,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/82ae508119f24f35a87627fd9d8d4dcb to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/82ae508119f24f35a87627fd9d8d4dcb 2024-12-09T08:16:40,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/a21c0c3357a143719aa32053a6b25f13 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/a21c0c3357a143719aa32053a6b25f13 2024-12-09T08:16:40,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/252721d79e294c0eba752af9bf0fd649 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/252721d79e294c0eba752af9bf0fd649 2024-12-09T08:16:40,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/af750702f4d8485e819e90192e218882 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/af750702f4d8485e819e90192e218882 2024-12-09T08:16:40,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ac131ec555d5455d8e57f64842aeda12 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/ac131ec555d5455d8e57f64842aeda12 2024-12-09T08:16:40,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/d2634520fa824b2c8cff217ee22e9c86 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/d2634520fa824b2c8cff217ee22e9c86 2024-12-09T08:16:40,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/219e88a305434a8080af786866ad7d96 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/219e88a305434a8080af786866ad7d96 2024-12-09T08:16:40,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/669658180efb4aa2a5e8e06cd9b91dde to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/669658180efb4aa2a5e8e06cd9b91dde 2024-12-09T08:16:40,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/c37422611ce948f1bff542023714f320 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/c37422611ce948f1bff542023714f320 2024-12-09T08:16:40,549 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/recovered.edits/430.seqid, newMaxSeqId=430, maxSeqId=1 2024-12-09T08:16:40,550 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. 2024-12-09T08:16:40,550 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 6b37acb92571896382ef9b774ba15dad: Waiting for close lock at 1733732199181Running coprocessor pre-close hooks at 1733732199181Disabling compacts and flushes for region at 1733732199181Disabling writes for close at 1733732199182 (+1 ms)Obtaining lock to block concurrent updates at 1733732199182Preparing flush snapshotting stores in 6b37acb92571896382ef9b774ba15dad at 1733732199182Finished memstore snapshotting TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733732199182Flushing stores of TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad. at 1733732199183 (+1 ms)Flushing 6b37acb92571896382ef9b774ba15dad/A: creating writer at 1733732199183Flushing 6b37acb92571896382ef9b774ba15dad/A: appending metadata at 1733732199186 (+3 ms)Flushing 6b37acb92571896382ef9b774ba15dad/A: closing flushed file at 1733732199186Flushing 6b37acb92571896382ef9b774ba15dad/B: creating writer at 1733732199601 (+415 ms)Flushing 6b37acb92571896382ef9b774ba15dad/B: appending metadata at 1733732199604 (+3 ms)Flushing 6b37acb92571896382ef9b774ba15dad/B: closing flushed file at 1733732199604Flushing 6b37acb92571896382ef9b774ba15dad/C: creating writer at 1733732200018 (+414 ms)Flushing 6b37acb92571896382ef9b774ba15dad/C: appending metadata at 1733732200022 (+4 ms)Flushing 6b37acb92571896382ef9b774ba15dad/C: closing flushed file at 1733732200022Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@738224c3: reopening flushed file at 1733732200437 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58a433bb: reopening flushed file at 1733732200445 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35dc7ab1: reopening flushed file at 1733732200450 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6b37acb92571896382ef9b774ba15dad in 1274ms, sequenceid=427, compaction requested=true at 1733732200456 (+6 ms)Writing region close event to WAL at 1733732200546 (+90 ms)Running coprocessor post-close hooks at 1733732200550 (+4 ms)Closed at 1733732200550 2024-12-09T08:16:40,553 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=6b37acb92571896382ef9b774ba15dad, regionState=CLOSED 2024-12-09T08:16:40,554 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:40,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:16:40,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-09T08:16:40,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 6b37acb92571896382ef9b774ba15dad, server=5bda946059d1,43255,1733732044053 in 1.5260 sec 2024-12-09T08:16:40,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-09T08:16:40,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b37acb92571896382ef9b774ba15dad, UNASSIGN in 1.5320 sec 2024-12-09T08:16:40,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-09T08:16:40,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5360 sec 2024-12-09T08:16:40,565 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732200564"}]},"ts":"1733732200564"} 2024-12-09T08:16:40,566 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T08:16:40,566 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T08:16:40,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5500 sec 2024-12-09T08:16:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T08:16:41,171 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:41,171 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T08:16:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:41,173 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T08:16:41,174 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:41,176 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:41,178 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/recovered.edits] 2024-12-09T08:16:41,181 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/143c10341e7a47748d47939979521832 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/143c10341e7a47748d47939979521832 2024-12-09T08:16:41,182 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/28d57ef803964eda91570a59d0abfcb1 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/28d57ef803964eda91570a59d0abfcb1 2024-12-09T08:16:41,183 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8c3cc0a5f37e49e093ce99f0ccaa28e6 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/A/8c3cc0a5f37e49e093ce99f0ccaa28e6 2024-12-09T08:16:41,186 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/3b30614439664c699e37f7de4f6672d7 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/3b30614439664c699e37f7de4f6672d7 2024-12-09T08:16:41,187 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/bbb115cebddd42eab35b339a76c6ad7a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/bbb115cebddd42eab35b339a76c6ad7a 2024-12-09T08:16:41,188 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/eab245f6f54e4bee823946fb39e5ecfe to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/B/eab245f6f54e4bee823946fb39e5ecfe 2024-12-09T08:16:41,190 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/0e7b231af49a4a219badbd7b598a9186 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/0e7b231af49a4a219badbd7b598a9186 2024-12-09T08:16:41,191 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/21df66695d664f47aa4b5465eefd813a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/21df66695d664f47aa4b5465eefd813a 2024-12-09T08:16:41,192 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/d3da9f7ee0b842dab248e2e8b8b44ca3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/C/d3da9f7ee0b842dab248e2e8b8b44ca3 2024-12-09T08:16:41,194 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/recovered.edits/430.seqid to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad/recovered.edits/430.seqid 2024-12-09T08:16:41,194 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/6b37acb92571896382ef9b774ba15dad 2024-12-09T08:16:41,194 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T08:16:41,196 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:41,198 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T08:16:41,200 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T08:16:41,201 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:41,201 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T08:16:41,201 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733732201201"}]},"ts":"9223372036854775807"} 2024-12-09T08:16:41,203 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T08:16:41,203 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6b37acb92571896382ef9b774ba15dad, NAME => 'TestAcidGuarantees,,1733732171452.6b37acb92571896382ef9b774ba15dad.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T08:16:41,203 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T08:16:41,204 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733732201203"}]},"ts":"9223372036854775807"} 2024-12-09T08:16:41,205 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T08:16:41,206 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:41,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 35 msec 2024-12-09T08:16:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T08:16:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T08:16:41,284 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:41,294 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 248) - Thread LEAK? -, OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=421 (was 487), ProcessCount=11 (was 11), AvailableMemoryMB=4444 (was 4486) 2024-12-09T08:16:41,303 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=421, ProcessCount=11, AvailableMemoryMB=4443 2024-12-09T08:16:41,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T08:16:41,305 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:16:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:41,306 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T08:16:41,307 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:41,307 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-12-09T08:16:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T08:16:41,307 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T08:16:41,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742351_1527 (size=1061) 2024-12-09T08:16:41,313 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447 2024-12-09T08:16:41,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742352_1528 (size=53) 2024-12-09T08:16:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T08:16:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T08:16:41,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:16:41,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 19c9630e7ab12af49561314a634ad52e, disabling compactions & flushes 2024-12-09T08:16:41,720 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:41,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:41,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. after waiting 0 ms 2024-12-09T08:16:41,721 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:41,721 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:41,721 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 19c9630e7ab12af49561314a634ad52e: Waiting for close lock at 1733732201720Disabling compacts and flushes for region at 1733732201720Disabling writes for close at 1733732201720Writing region close event to WAL at 1733732201721 (+1 ms)Closed at 1733732201721 2024-12-09T08:16:41,724 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T08:16:41,724 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733732201724"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733732201724"}]},"ts":"1733732201724"} 2024-12-09T08:16:41,726 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T08:16:41,727 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T08:16:41,728 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732201727"}]},"ts":"1733732201727"} 2024-12-09T08:16:41,729 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T08:16:41,730 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, ASSIGN}] 2024-12-09T08:16:41,731 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, ASSIGN 2024-12-09T08:16:41,732 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, ASSIGN; state=OFFLINE, location=5bda946059d1,43255,1733732044053; forceNewPlan=false, retain=false 2024-12-09T08:16:41,883 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=19c9630e7ab12af49561314a634ad52e, regionState=OPENING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:16:41,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, ASSIGN because future has completed 2024-12-09T08:16:41,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:16:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T08:16:42,045 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:42,046 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:16:42,046 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,047 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:16:42,047 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,047 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,049 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,051 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:16:42,051 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19c9630e7ab12af49561314a634ad52e columnFamilyName A 2024-12-09T08:16:42,052 DEBUG [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:42,052 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.HStore(327): Store=19c9630e7ab12af49561314a634ad52e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:16:42,052 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,053 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:16:42,053 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19c9630e7ab12af49561314a634ad52e columnFamilyName B 2024-12-09T08:16:42,053 DEBUG [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:42,053 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.HStore(327): Store=19c9630e7ab12af49561314a634ad52e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:16:42,054 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,054 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:16:42,054 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19c9630e7ab12af49561314a634ad52e columnFamilyName C 2024-12-09T08:16:42,054 DEBUG [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:42,055 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.HStore(327): Store=19c9630e7ab12af49561314a634ad52e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:16:42,055 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:42,055 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,055 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,056 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,056 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,056 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,057 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:16:42,057 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,058 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:16:42,059 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 19c9630e7ab12af49561314a634ad52e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60054149, jitterRate=-0.10512344539165497}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:16:42,059 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:42,059 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 19c9630e7ab12af49561314a634ad52e: Running coprocessor pre-open hook at 1733732202047Writing region info on filesystem at 1733732202047Initializing all the Stores at 1733732202048 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732202049 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732202049Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732202049Cleaning up temporary data from old regions at 1733732202056 (+7 ms)Running coprocessor post-open hooks at 1733732202059 (+3 ms)Region opened successfully at 1733732202059 2024-12-09T08:16:42,060 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., pid=140, masterSystemTime=1733732202039 2024-12-09T08:16:42,061 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:42,061 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:42,061 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=19c9630e7ab12af49561314a634ad52e, regionState=OPEN, openSeqNum=2, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:16:42,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:16:42,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-09T08:16:42,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 in 177 msec 2024-12-09T08:16:42,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-09T08:16:42,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, ASSIGN in 334 msec 2024-12-09T08:16:42,066 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T08:16:42,066 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732202066"}]},"ts":"1733732202066"} 2024-12-09T08:16:42,066 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T08:16:42,067 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T08:16:42,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 762 msec 2024-12-09T08:16:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T08:16:42,443 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:42,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cec06c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:42,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:42,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:42,447 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d7d706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:42,449 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63b4c66b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:42,450 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:42,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:42,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:42,453 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T08:16:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 5bda946059d1,45611,1733732043366 2024-12-09T08:16:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@48edd470 2024-12-09T08:16:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T08:16:42,455 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T08:16:42,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T08:16:42,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:16:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T08:16:42,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742353_1529 (size=1097) 2024-12-09T08:16:42,872 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T08:16:42,873 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T08:16:42,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T08:16:42,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, REOPEN/MOVE}] 2024-12-09T08:16:42,882 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, REOPEN/MOVE 2024-12-09T08:16:42,883 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=19c9630e7ab12af49561314a634ad52e, regionState=CLOSING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:16:42,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, REOPEN/MOVE because future has completed 2024-12-09T08:16:42,884 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:16:42,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:16:43,038 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,038 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:16:43,038 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 19c9630e7ab12af49561314a634ad52e, disabling compactions & flushes 2024-12-09T08:16:43,038 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,038 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,038 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. after waiting 0 ms 2024-12-09T08:16:43,039 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,047 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T08:16:43,048 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,048 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 19c9630e7ab12af49561314a634ad52e: Waiting for close lock at 1733732203038Running coprocessor pre-close hooks at 1733732203038Disabling compacts and flushes for region at 1733732203038Disabling writes for close at 1733732203038Writing region close event to WAL at 1733732203039 (+1 ms)Running coprocessor post-close hooks at 1733732203048 (+9 ms)Closed at 1733732203048 2024-12-09T08:16:43,048 WARN [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 19c9630e7ab12af49561314a634ad52e to self. 2024-12-09T08:16:43,050 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,051 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=19c9630e7ab12af49561314a634ad52e, regionState=CLOSED 2024-12-09T08:16:43,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:16:43,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-09T08:16:43,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 in 169 msec 2024-12-09T08:16:43,056 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, REOPEN/MOVE; state=CLOSED, location=5bda946059d1,43255,1733732044053; forceNewPlan=false, retain=true 2024-12-09T08:16:43,208 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=19c9630e7ab12af49561314a634ad52e, regionState=OPENING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, REOPEN/MOVE because future has completed 2024-12-09T08:16:43,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:16:43,372 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,372 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:16:43,373 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,373 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:16:43,373 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,373 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,374 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,375 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:16:43,375 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19c9630e7ab12af49561314a634ad52e columnFamilyName A 2024-12-09T08:16:43,376 DEBUG [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:43,377 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.HStore(327): Store=19c9630e7ab12af49561314a634ad52e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:16:43,377 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,378 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:16:43,378 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19c9630e7ab12af49561314a634ad52e columnFamilyName B 2024-12-09T08:16:43,378 DEBUG [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:43,378 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.HStore(327): Store=19c9630e7ab12af49561314a634ad52e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:16:43,379 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,379 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T08:16:43,379 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19c9630e7ab12af49561314a634ad52e columnFamilyName C 2024-12-09T08:16:43,379 DEBUG [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:43,380 INFO [StoreOpener-19c9630e7ab12af49561314a634ad52e-1 {}] regionserver.HStore(327): Store=19c9630e7ab12af49561314a634ad52e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:16:43,380 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,380 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,380 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,382 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,383 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,383 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,383 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:16:43,385 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,386 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 19c9630e7ab12af49561314a634ad52e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59206687, jitterRate=-0.11775161325931549}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:16:43,386 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,387 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 19c9630e7ab12af49561314a634ad52e: Running coprocessor pre-open hook at 1733732203373Writing region info on filesystem at 1733732203373Initializing all the Stores at 1733732203374 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732203374Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732203374Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732203374Cleaning up temporary data from old regions at 1733732203383 (+9 ms)Running coprocessor post-open hooks at 1733732203386 (+3 ms)Region opened successfully at 1733732203387 (+1 ms) 2024-12-09T08:16:43,388 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., pid=145, masterSystemTime=1733732203366 2024-12-09T08:16:43,390 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,390 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,391 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=19c9630e7ab12af49561314a634ad52e, regionState=OPEN, openSeqNum=5, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:16:43,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-12-09T08:16:43,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 in 181 msec 2024-12-09T08:16:43,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-09T08:16:43,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, REOPEN/MOVE in 516 msec 2024-12-09T08:16:43,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-09T08:16:43,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2024-12-09T08:16:43,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 944 msec 2024-12-09T08:16:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-09T08:16:43,406 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:43,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:43,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:43,408 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17007899, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:43,410 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe4c6ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:43,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:43,414 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56daf51a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:43,416 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:43,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc39a52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:43,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:43,419 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8087f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:43,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:43,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:43,421 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:43,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ea2e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:43,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:43,423 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17a894c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:43,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:43,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:43,425 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:43,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a8c1c19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:43,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:43,427 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47fc084d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:43,429 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:43,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c7af9bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:43,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:43,431 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@362f3df7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:43,432 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f3a9a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:43,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:43,434 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@141ecb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:43,436 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:43,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740318be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:43,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:43,438 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e7f3bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:43,439 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323e9a54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:43,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:43,441 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ffc898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:43,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:43,442 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447cadff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,45611,-1 for getting cluster id 2024-12-09T08:16:43,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:16:43,444 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f412b64-5200-4500-97f2-3b4b9d069ff7' 2024-12-09T08:16:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:16:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f412b64-5200-4500-97f2-3b4b9d069ff7" 2024-12-09T08:16:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50dda203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,45611,-1] 2024-12-09T08:16:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:16:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:16:43,445 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:16:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a3f4c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:16:43,446 DEBUG [Thread-2378 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:43,446 DEBUG [Thread-2363 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:43,446 DEBUG [Thread-2365 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:43,446 DEBUG [Thread-2380 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:43,446 DEBUG [Thread-2371 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:43,446 DEBUG [Thread-2374 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:43,447 DEBUG [Thread-2367 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:43,447 DEBUG [Thread-2369 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:43,447 DEBUG [Thread-2376 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:43,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:43,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:43,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:43,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:43,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:43,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:43,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:43,448 DEBUG [Thread-2382 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:16:43,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:43,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:43,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-09T08:16:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,43255,1733732044053, seqNum=-1] 2024-12-09T08:16:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:16:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:16:43,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:16:43,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:43,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:43,451 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:16:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:16:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:16:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:16:43,452 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:43,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T08:16:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:16:43,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:16:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:16:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:16:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5] 2024-12-09T08:16:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:16:43,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:43,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:43,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:43,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:43,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:43,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:43,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732263477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732263477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732263481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732263481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120943994c13b1fb42a0a2d6769fc177ffd5_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732203458/Put/seqid=0 2024-12-09T08:16:43,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732263486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742354_1530 (size=12154) 2024-12-09T08:16:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T08:16:43,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732263591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732263591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732263591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732263591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732263591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T08:16:43,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:43,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:43,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:43,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T08:16:43,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:43,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:43,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T08:16:43,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732263801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732263801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732263801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732263801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732263801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:43,895 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:43,900 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120943994c13b1fb42a0a2d6769fc177ffd5_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120943994c13b1fb42a0a2d6769fc177ffd5_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:43,902 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/59e425a888b64f518bebe8d783f17346, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:43,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/59e425a888b64f518bebe8d783f17346 is 175, key is test_row_0/A:col10/1733732203458/Put/seqid=0 2024-12-09T08:16:43,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742355_1531 (size=30955) 2024-12-09T08:16:43,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/59e425a888b64f518bebe8d783f17346 2024-12-09T08:16:43,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T08:16:43,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:43,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:43,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:43,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:43,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/7172f2786db343d3b3ac8c7f9853a652 is 50, key is test_row_0/B:col10/1733732203458/Put/seqid=0 2024-12-09T08:16:43,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742356_1532 (size=12001) 2024-12-09T08:16:43,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/7172f2786db343d3b3ac8c7f9853a652 2024-12-09T08:16:43,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/857ed0af0f894e11a2629b62bd06942d is 50, key is test_row_0/C:col10/1733732203458/Put/seqid=0 2024-12-09T08:16:43,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742357_1533 (size=12001) 2024-12-09T08:16:44,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T08:16:44,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:44,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:44,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T08:16:44,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732264111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732264112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732264112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732264112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732264112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T08:16:44,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:44,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:44,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:44,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:44,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:44,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T08:16:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:44,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:44,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/857ed0af0f894e11a2629b62bd06942d 2024-12-09T08:16:44,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/59e425a888b64f518bebe8d783f17346 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/59e425a888b64f518bebe8d783f17346 2024-12-09T08:16:44,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/59e425a888b64f518bebe8d783f17346, entries=150, sequenceid=16, filesize=30.2 K 2024-12-09T08:16:44,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/7172f2786db343d3b3ac8c7f9853a652 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/7172f2786db343d3b3ac8c7f9853a652 2024-12-09T08:16:44,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/7172f2786db343d3b3ac8c7f9853a652, entries=150, sequenceid=16, filesize=11.7 K 2024-12-09T08:16:44,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/857ed0af0f894e11a2629b62bd06942d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/857ed0af0f894e11a2629b62bd06942d 2024-12-09T08:16:44,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/857ed0af0f894e11a2629b62bd06942d, entries=150, sequenceid=16, filesize=11.7 K 2024-12-09T08:16:44,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 19c9630e7ab12af49561314a634ad52e in 941ms, sequenceid=16, compaction requested=false 2024-12-09T08:16:44,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:44,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T08:16:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:44,523 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T08:16:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:44,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d35dad6468b94776949a77bbc4844d75_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732203480/Put/seqid=0 2024-12-09T08:16:44,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742358_1534 (size=12154) 2024-12-09T08:16:44,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:44,538 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d35dad6468b94776949a77bbc4844d75_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d35dad6468b94776949a77bbc4844d75_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:44,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/bef94e8ba03945e08c48e5ba6ae40800, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:44,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/bef94e8ba03945e08c48e5ba6ae40800 is 175, key is test_row_0/A:col10/1733732203480/Put/seqid=0 2024-12-09T08:16:44,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742359_1535 (size=30955) 2024-12-09T08:16:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T08:16:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:44,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732264631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732264631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732264632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732264632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732264633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732264741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732264741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732264741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732264741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732264741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,942 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/bef94e8ba03945e08c48e5ba6ae40800 2024-12-09T08:16:44,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/bf4903566eb446edbfc51cac3893221e is 50, key is test_row_0/B:col10/1733732203480/Put/seqid=0 2024-12-09T08:16:44,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742360_1536 (size=12001) 2024-12-09T08:16:44,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732264951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732264951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732264951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,953 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/bf4903566eb446edbfc51cac3893221e 2024-12-09T08:16:44,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732264951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:44,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732264951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:44,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/8f174d9ba24b4ca0b5e1505f2fcbd605 is 50, key is test_row_0/C:col10/1733732203480/Put/seqid=0 2024-12-09T08:16:44,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742361_1537 (size=12001) 2024-12-09T08:16:44,970 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/8f174d9ba24b4ca0b5e1505f2fcbd605 2024-12-09T08:16:44,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/bef94e8ba03945e08c48e5ba6ae40800 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/bef94e8ba03945e08c48e5ba6ae40800 2024-12-09T08:16:44,976 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/bef94e8ba03945e08c48e5ba6ae40800, entries=150, sequenceid=36, filesize=30.2 K 2024-12-09T08:16:44,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/bf4903566eb446edbfc51cac3893221e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/bf4903566eb446edbfc51cac3893221e 2024-12-09T08:16:44,979 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/bf4903566eb446edbfc51cac3893221e, entries=150, sequenceid=36, filesize=11.7 K 2024-12-09T08:16:44,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/8f174d9ba24b4ca0b5e1505f2fcbd605 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/8f174d9ba24b4ca0b5e1505f2fcbd605 2024-12-09T08:16:44,983 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/8f174d9ba24b4ca0b5e1505f2fcbd605, entries=150, sequenceid=36, filesize=11.7 K 2024-12-09T08:16:44,984 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 19c9630e7ab12af49561314a634ad52e in 461ms, sequenceid=36, compaction requested=false 2024-12-09T08:16:44,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:44,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:44,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-09T08:16:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-09T08:16:44,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-09T08:16:44,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5330 sec 2024-12-09T08:16:44,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.5390 sec 2024-12-09T08:16:45,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:45,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T08:16:45,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:45,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:45,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:45,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:45,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:45,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:45,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ea724410bf4e49db9366feee0ee264a3_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732205264/Put/seqid=0 2024-12-09T08:16:45,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742362_1538 (size=14594) 2024-12-09T08:16:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732265276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732265277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732265276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732265280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732265280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732265391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732265391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732265391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732265391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732265391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T08:16:45,601 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:45,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-09T08:16:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T08:16:45,603 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:45,603 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:45,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732265601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732265601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732265601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732265602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732265602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:45,676 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ea724410bf4e49db9366feee0ee264a3_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ea724410bf4e49db9366feee0ee264a3_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:45,677 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/d31915d9ff96406bb32a259408b35c0c, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:45,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/d31915d9ff96406bb32a259408b35c0c is 175, key is test_row_0/A:col10/1733732205264/Put/seqid=0 2024-12-09T08:16:45,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742363_1539 (size=39549) 2024-12-09T08:16:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T08:16:45,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T08:16:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:45,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:45,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:45,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T08:16:45,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:45,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:45,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:45,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:45,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:45,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732265911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732265911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732265911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732265911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732265911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T08:16:46,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T08:16:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,069 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:16:46,097 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/d31915d9ff96406bb32a259408b35c0c 2024-12-09T08:16:46,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/2a3921f549574652b8976b83025875a3 is 50, key is test_row_0/B:col10/1733732205264/Put/seqid=0 2024-12-09T08:16:46,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742364_1540 (size=12001) 2024-12-09T08:16:46,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T08:16:46,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:46,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T08:16:46,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T08:16:46,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:46,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732266421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:46,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732266421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:46,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732266421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:46,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732266421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:46,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732266421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:46,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/2a3921f549574652b8976b83025875a3 2024-12-09T08:16:46,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T08:16:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/f3f4ea13ae0f44e597c67133bce8533e is 50, key is test_row_0/C:col10/1733732205264/Put/seqid=0 2024-12-09T08:16:46,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742365_1541 (size=12001) 2024-12-09T08:16:46,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T08:16:46,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:46,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T08:16:46,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T08:16:46,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:46,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:46,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/f3f4ea13ae0f44e597c67133bce8533e 2024-12-09T08:16:46,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/d31915d9ff96406bb32a259408b35c0c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/d31915d9ff96406bb32a259408b35c0c 2024-12-09T08:16:46,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/d31915d9ff96406bb32a259408b35c0c, entries=200, sequenceid=53, filesize=38.6 K 2024-12-09T08:16:46,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/2a3921f549574652b8976b83025875a3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/2a3921f549574652b8976b83025875a3 2024-12-09T08:16:46,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/2a3921f549574652b8976b83025875a3, entries=150, sequenceid=53, filesize=11.7 K 2024-12-09T08:16:46,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/f3f4ea13ae0f44e597c67133bce8533e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/f3f4ea13ae0f44e597c67133bce8533e 2024-12-09T08:16:46,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/f3f4ea13ae0f44e597c67133bce8533e, entries=150, sequenceid=53, filesize=11.7 K 2024-12-09T08:16:46,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 19c9630e7ab12af49561314a634ad52e in 1680ms, sequenceid=53, compaction requested=true 2024-12-09T08:16:46,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:46,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:46,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:46,945 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:46,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:46,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:46,945 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:46,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:46,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:46,945 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:46,945 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:46,945 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/A is initiating minor compaction (all files) 2024-12-09T08:16:46,945 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/B is initiating minor compaction (all files) 2024-12-09T08:16:46,946 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/A in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,946 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/B in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,946 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/59e425a888b64f518bebe8d783f17346, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/bef94e8ba03945e08c48e5ba6ae40800, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/d31915d9ff96406bb32a259408b35c0c] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=99.1 K 2024-12-09T08:16:46,946 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/7172f2786db343d3b3ac8c7f9853a652, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/bf4903566eb446edbfc51cac3893221e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/2a3921f549574652b8976b83025875a3] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=35.2 K 2024-12-09T08:16:46,946 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,946 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/59e425a888b64f518bebe8d783f17346, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/bef94e8ba03945e08c48e5ba6ae40800, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/d31915d9ff96406bb32a259408b35c0c] 2024-12-09T08:16:46,946 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59e425a888b64f518bebe8d783f17346, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733732203456 2024-12-09T08:16:46,946 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 7172f2786db343d3b3ac8c7f9853a652, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733732203456 2024-12-09T08:16:46,946 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting bf4903566eb446edbfc51cac3893221e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733732203472 2024-12-09T08:16:46,946 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting bef94e8ba03945e08c48e5ba6ae40800, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733732203472 2024-12-09T08:16:46,947 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting d31915d9ff96406bb32a259408b35c0c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733732205263 2024-12-09T08:16:46,947 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a3921f549574652b8976b83025875a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733732205263 2024-12-09T08:16:46,951 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:46,952 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:46,952 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/5cde1e7a8b7f430e93a45872c151b69a is 50, key is test_row_0/B:col10/1733732205264/Put/seqid=0 2024-12-09T08:16:46,952 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209fe485ee4eed64c2a9dfba3ff85ef3d54_19c9630e7ab12af49561314a634ad52e store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:46,954 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209fe485ee4eed64c2a9dfba3ff85ef3d54_19c9630e7ab12af49561314a634ad52e, store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:46,954 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209fe485ee4eed64c2a9dfba3ff85ef3d54_19c9630e7ab12af49561314a634ad52e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:46,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742366_1542 (size=12104) 2024-12-09T08:16:46,962 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/5cde1e7a8b7f430e93a45872c151b69a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/5cde1e7a8b7f430e93a45872c151b69a 2024-12-09T08:16:46,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742367_1543 (size=4469) 2024-12-09T08:16:46,966 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/B of 19c9630e7ab12af49561314a634ad52e into 5cde1e7a8b7f430e93a45872c151b69a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:46,966 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:46,966 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/B, priority=13, startTime=1733732206945; duration=0sec 2024-12-09T08:16:46,966 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:46,967 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:B 2024-12-09T08:16:46,967 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:46,967 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:46,967 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/C is initiating minor compaction (all files) 2024-12-09T08:16:46,967 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/C in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,967 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/857ed0af0f894e11a2629b62bd06942d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/8f174d9ba24b4ca0b5e1505f2fcbd605, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/f3f4ea13ae0f44e597c67133bce8533e] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=35.2 K 2024-12-09T08:16:46,968 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 857ed0af0f894e11a2629b62bd06942d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733732203456 2024-12-09T08:16:46,968 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f174d9ba24b4ca0b5e1505f2fcbd605, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733732203472 2024-12-09T08:16:46,968 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f4ea13ae0f44e597c67133bce8533e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733732205263 2024-12-09T08:16:46,973 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#C#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:46,973 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/645c063d0f9f4807aa144cb90361ab2f is 50, key is test_row_0/C:col10/1733732205264/Put/seqid=0 2024-12-09T08:16:46,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742368_1544 (size=12104) 2024-12-09T08:16:46,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T08:16:46,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:46,978 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T08:16:46,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:46,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:46,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:46,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:46,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:46,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:46,983 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/645c063d0f9f4807aa144cb90361ab2f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/645c063d0f9f4807aa144cb90361ab2f 2024-12-09T08:16:46,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ea955e168aed4dcca6e7a9b1aa905ee1_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732205275/Put/seqid=0 2024-12-09T08:16:46,989 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/C of 19c9630e7ab12af49561314a634ad52e into 645c063d0f9f4807aa144cb90361ab2f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:46,990 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:46,990 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/C, priority=13, startTime=1733732206945; duration=0sec 2024-12-09T08:16:46,990 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:46,990 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:C 2024-12-09T08:16:46,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742369_1545 (size=12154) 2024-12-09T08:16:47,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:47,004 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ea955e168aed4dcca6e7a9b1aa905ee1_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ea955e168aed4dcca6e7a9b1aa905ee1_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:47,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/9e443f9226004a17baea8c6d91e46329, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:47,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/9e443f9226004a17baea8c6d91e46329 is 175, key is test_row_0/A:col10/1733732205275/Put/seqid=0 2024-12-09T08:16:47,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742370_1546 (size=30955) 2024-12-09T08:16:47,011 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/9e443f9226004a17baea8c6d91e46329 2024-12-09T08:16:47,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/8f7d5c853c7e488ca77b48ae01b067e4 is 50, key is test_row_0/B:col10/1733732205275/Put/seqid=0 2024-12-09T08:16:47,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742371_1547 (size=12001) 2024-12-09T08:16:47,363 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#A#compaction#465 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:47,364 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/7d29fd9976444a64b76d09704a2ad233 is 175, key is test_row_0/A:col10/1733732205264/Put/seqid=0 2024-12-09T08:16:47,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742372_1548 (size=31058) 2024-12-09T08:16:47,431 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/8f7d5c853c7e488ca77b48ae01b067e4 2024-12-09T08:16:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:47,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:47,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/8f004061cba5402a8e5eee57bc7126b3 is 50, key is test_row_0/C:col10/1733732205275/Put/seqid=0 2024-12-09T08:16:47,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742373_1549 (size=12001) 2024-12-09T08:16:47,440 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/8f004061cba5402a8e5eee57bc7126b3 2024-12-09T08:16:47,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/9e443f9226004a17baea8c6d91e46329 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/9e443f9226004a17baea8c6d91e46329 2024-12-09T08:16:47,447 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/9e443f9226004a17baea8c6d91e46329, entries=150, sequenceid=70, filesize=30.2 K 2024-12-09T08:16:47,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/8f7d5c853c7e488ca77b48ae01b067e4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/8f7d5c853c7e488ca77b48ae01b067e4 2024-12-09T08:16:47,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732267444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732267445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,450 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/8f7d5c853c7e488ca77b48ae01b067e4, entries=150, sequenceid=70, filesize=11.7 K 2024-12-09T08:16:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/8f004061cba5402a8e5eee57bc7126b3 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/8f004061cba5402a8e5eee57bc7126b3 2024-12-09T08:16:47,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732267446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732267448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732267448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,454 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/8f004061cba5402a8e5eee57bc7126b3, entries=150, sequenceid=70, filesize=11.7 K 2024-12-09T08:16:47,454 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 19c9630e7ab12af49561314a634ad52e in 476ms, sequenceid=70, compaction requested=false 2024-12-09T08:16:47,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:47,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:47,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-09T08:16:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-09T08:16:47,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-09T08:16:47,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8520 sec 2024-12-09T08:16:47,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.8560 sec 2024-12-09T08:16:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:47,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T08:16:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:47,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095f3d9a11e12c4f6dbafdcd09240f2af1_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732207446/Put/seqid=0 2024-12-09T08:16:47,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742374_1550 (size=14594) 2024-12-09T08:16:47,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732267568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732267571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732267571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732267572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732267572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732267681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732267681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732267681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732267681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732267681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T08:16:47,761 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:47,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-09T08:16:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T08:16:47,763 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:47,764 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:47,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:47,770 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/7d29fd9976444a64b76d09704a2ad233 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/7d29fd9976444a64b76d09704a2ad233 2024-12-09T08:16:47,773 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/A of 19c9630e7ab12af49561314a634ad52e into 7d29fd9976444a64b76d09704a2ad233(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:47,773 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:47,773 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/A, priority=13, startTime=1733732206945; duration=0sec 2024-12-09T08:16:47,774 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:47,774 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:A 2024-12-09T08:16:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T08:16:47,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732267891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732267891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732267891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732267891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732267891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:47,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T08:16:47,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:47,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:47,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:47,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:47,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:47,962 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:47,965 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095f3d9a11e12c4f6dbafdcd09240f2af1_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095f3d9a11e12c4f6dbafdcd09240f2af1_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:47,965 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/629974eb0f234d839358d799675b874f, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:47,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/629974eb0f234d839358d799675b874f is 175, key is test_row_0/A:col10/1733732207446/Put/seqid=0 2024-12-09T08:16:47,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742375_1551 (size=39549) 2024-12-09T08:16:48,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T08:16:48,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:48,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T08:16:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732268201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732268201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732268201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732268201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732268201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:48,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T08:16:48,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:48,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,369 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/629974eb0f234d839358d799675b874f 2024-12-09T08:16:48,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T08:16:48,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:48,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/55fc59b65f3e4ca0bb1df87a621cba6a is 50, key is test_row_0/B:col10/1733732207446/Put/seqid=0 2024-12-09T08:16:48,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742376_1552 (size=12001) 2024-12-09T08:16:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T08:16:48,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T08:16:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T08:16:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732268711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:48,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732268711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:48,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732268711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732268712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732268712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:48,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/55fc59b65f3e4ca0bb1df87a621cba6a 2024-12-09T08:16:48,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/b9811d4fed08470fb678379f7bc9c4bb is 50, key is test_row_0/C:col10/1733732207446/Put/seqid=0 2024-12-09T08:16:48,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742377_1553 (size=12001) 2024-12-09T08:16:48,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T08:16:48,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:48,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T08:16:48,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T08:16:48,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:48,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:48,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:49,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T08:16:49,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:49,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:49,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:49,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:49,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:49,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/b9811d4fed08470fb678379f7bc9c4bb 2024-12-09T08:16:49,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/629974eb0f234d839358d799675b874f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/629974eb0f234d839358d799675b874f 2024-12-09T08:16:49,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/629974eb0f234d839358d799675b874f, entries=200, sequenceid=89, filesize=38.6 K 2024-12-09T08:16:49,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/55fc59b65f3e4ca0bb1df87a621cba6a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/55fc59b65f3e4ca0bb1df87a621cba6a 2024-12-09T08:16:49,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/55fc59b65f3e4ca0bb1df87a621cba6a, entries=150, sequenceid=89, filesize=11.7 K 2024-12-09T08:16:49,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/b9811d4fed08470fb678379f7bc9c4bb as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/b9811d4fed08470fb678379f7bc9c4bb 2024-12-09T08:16:49,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/b9811d4fed08470fb678379f7bc9c4bb, entries=150, sequenceid=89, filesize=11.7 K 2024-12-09T08:16:49,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 19c9630e7ab12af49561314a634ad52e in 1648ms, sequenceid=89, compaction requested=true 2024-12-09T08:16:49,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:49,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:49,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:49,202 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:49,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:49,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:49,202 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:49,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:49,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:49,202 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:49,202 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:49,203 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/B is initiating minor compaction (all files) 2024-12-09T08:16:49,203 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/A is initiating minor compaction (all files) 2024-12-09T08:16:49,203 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/B in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:49,203 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/A in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:49,203 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/5cde1e7a8b7f430e93a45872c151b69a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/8f7d5c853c7e488ca77b48ae01b067e4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/55fc59b65f3e4ca0bb1df87a621cba6a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=35.3 K 2024-12-09T08:16:49,203 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/7d29fd9976444a64b76d09704a2ad233, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/9e443f9226004a17baea8c6d91e46329, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/629974eb0f234d839358d799675b874f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=99.2 K 2024-12-09T08:16:49,203 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:49,203 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/7d29fd9976444a64b76d09704a2ad233, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/9e443f9226004a17baea8c6d91e46329, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/629974eb0f234d839358d799675b874f] 2024-12-09T08:16:49,203 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cde1e7a8b7f430e93a45872c151b69a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733732205263 2024-12-09T08:16:49,203 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d29fd9976444a64b76d09704a2ad233, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733732205263 2024-12-09T08:16:49,203 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f7d5c853c7e488ca77b48ae01b067e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733732205275 2024-12-09T08:16:49,203 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e443f9226004a17baea8c6d91e46329, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733732205275 2024-12-09T08:16:49,204 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 55fc59b65f3e4ca0bb1df87a621cba6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733732207445 2024-12-09T08:16:49,204 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 629974eb0f234d839358d799675b874f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733732207445 2024-12-09T08:16:49,208 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:49,210 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412090397fccca9894d6ab09d2d68ea632bb6_19c9630e7ab12af49561314a634ad52e store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:49,210 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:49,211 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412090397fccca9894d6ab09d2d68ea632bb6_19c9630e7ab12af49561314a634ad52e, store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:49,211 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/74a659e4520d4e1596aa019edf38b18c is 50, key is test_row_0/B:col10/1733732207446/Put/seqid=0 2024-12-09T08:16:49,211 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090397fccca9894d6ab09d2d68ea632bb6_19c9630e7ab12af49561314a634ad52e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:49,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742378_1554 (size=4469) 2024-12-09T08:16:49,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742379_1555 (size=12207) 2024-12-09T08:16:49,221 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/74a659e4520d4e1596aa019edf38b18c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/74a659e4520d4e1596aa019edf38b18c 2024-12-09T08:16:49,225 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/B of 19c9630e7ab12af49561314a634ad52e into 74a659e4520d4e1596aa019edf38b18c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:49,225 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:49,225 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/B, priority=13, startTime=1733732209202; duration=0sec 2024-12-09T08:16:49,226 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:49,226 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:B 2024-12-09T08:16:49,226 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:49,227 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:49,227 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/C is initiating minor compaction (all files) 2024-12-09T08:16:49,227 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/C in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:49,227 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/645c063d0f9f4807aa144cb90361ab2f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/8f004061cba5402a8e5eee57bc7126b3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/b9811d4fed08470fb678379f7bc9c4bb] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=35.3 K 2024-12-09T08:16:49,227 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 645c063d0f9f4807aa144cb90361ab2f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733732205263 2024-12-09T08:16:49,227 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f004061cba5402a8e5eee57bc7126b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733732205275 2024-12-09T08:16:49,228 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting b9811d4fed08470fb678379f7bc9c4bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733732207445 2024-12-09T08:16:49,234 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#C#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:49,235 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/db64cac4a61e48ccaea3ad951832c048 is 50, key is test_row_0/C:col10/1733732207446/Put/seqid=0 2024-12-09T08:16:49,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742380_1556 (size=12207) 2024-12-09T08:16:49,241 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/db64cac4a61e48ccaea3ad951832c048 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/db64cac4a61e48ccaea3ad951832c048 2024-12-09T08:16:49,245 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/C of 19c9630e7ab12af49561314a634ad52e into db64cac4a61e48ccaea3ad951832c048(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:49,245 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:49,245 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/C, priority=13, startTime=1733732209202; duration=0sec 2024-12-09T08:16:49,245 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:49,245 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:C 2024-12-09T08:16:49,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T08:16:49,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:49,287 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T08:16:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:49,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d5e6715da1264af2a2a52e952f724757_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732207571/Put/seqid=0 2024-12-09T08:16:49,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742381_1557 (size=12154) 2024-12-09T08:16:49,617 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#A#compaction#473 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:49,617 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/2ad158fac58e435e8f28a90c4df541c5 is 175, key is test_row_0/A:col10/1733732207446/Put/seqid=0 2024-12-09T08:16:49,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742382_1558 (size=31161) 2024-12-09T08:16:49,635 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/2ad158fac58e435e8f28a90c4df541c5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/2ad158fac58e435e8f28a90c4df541c5 2024-12-09T08:16:49,638 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/A of 19c9630e7ab12af49561314a634ad52e into 2ad158fac58e435e8f28a90c4df541c5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:49,638 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:49,638 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/A, priority=13, startTime=1733732209202; duration=0sec 2024-12-09T08:16:49,638 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:49,638 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:A 2024-12-09T08:16:49,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:49,699 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d5e6715da1264af2a2a52e952f724757_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d5e6715da1264af2a2a52e952f724757_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/c7b45edc7e2c4ddb9d80e416f600f0b9, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:49,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/c7b45edc7e2c4ddb9d80e416f600f0b9 is 175, key is test_row_0/A:col10/1733732207571/Put/seqid=0 2024-12-09T08:16:49,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742383_1559 (size=30955) 2024-12-09T08:16:49,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:49,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732269735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:49,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:49,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732269736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732269736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732269737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732269737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:49,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732269841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:49,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732269841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:49,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732269841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:49,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732269851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:49,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732269851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T08:16:50,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732270050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732270051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732270051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732270061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732270061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,103 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/c7b45edc7e2c4ddb9d80e416f600f0b9 2024-12-09T08:16:50,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/49c02ae75e314c47a7e9e4ffb5ec9485 is 50, key is test_row_0/B:col10/1733732207571/Put/seqid=0 2024-12-09T08:16:50,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742384_1560 (size=12001) 2024-12-09T08:16:50,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732270361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732270361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732270361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732270371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732270371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,518 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/49c02ae75e314c47a7e9e4ffb5ec9485 2024-12-09T08:16:50,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/30254500f5674510835ba7c43ae62866 is 50, key is test_row_0/C:col10/1733732207571/Put/seqid=0 2024-12-09T08:16:50,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742385_1561 (size=12001) 2024-12-09T08:16:50,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732270871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732270871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732270871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732270881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732270881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:50,928 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/30254500f5674510835ba7c43ae62866 2024-12-09T08:16:50,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/c7b45edc7e2c4ddb9d80e416f600f0b9 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/c7b45edc7e2c4ddb9d80e416f600f0b9 2024-12-09T08:16:50,934 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/c7b45edc7e2c4ddb9d80e416f600f0b9, entries=150, sequenceid=107, filesize=30.2 K 2024-12-09T08:16:50,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/49c02ae75e314c47a7e9e4ffb5ec9485 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/49c02ae75e314c47a7e9e4ffb5ec9485 2024-12-09T08:16:50,937 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/49c02ae75e314c47a7e9e4ffb5ec9485, entries=150, sequenceid=107, filesize=11.7 K 2024-12-09T08:16:50,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/30254500f5674510835ba7c43ae62866 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/30254500f5674510835ba7c43ae62866 2024-12-09T08:16:50,940 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/30254500f5674510835ba7c43ae62866, entries=150, sequenceid=107, filesize=11.7 K 2024-12-09T08:16:50,941 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 19c9630e7ab12af49561314a634ad52e in 1653ms, sequenceid=107, compaction requested=false 2024-12-09T08:16:50,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:50,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:50,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-09T08:16:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-09T08:16:50,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-09T08:16:50,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1780 sec 2024-12-09T08:16:50,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 3.1810 sec 2024-12-09T08:16:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:51,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T08:16:51,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:51,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:51,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:51,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:51,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:51,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:51,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091777a717815546db95c32600156be55c_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732211893/Put/seqid=0 2024-12-09T08:16:51,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732271901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:51,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732271901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:51,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742386_1562 (size=14644) 2024-12-09T08:16:51,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732271906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:51,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732271907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:51,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732271907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T08:16:51,931 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:51,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-09T08:16:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T08:16:51,933 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:51,934 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:51,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:52,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732272011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732272011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732272021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732272021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732272021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T08:16:52,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T08:16:52,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:52,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:52,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:52,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732272221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732272221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732272231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732272231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732272231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T08:16:52,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:52,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:52,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:52,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T08:16:52,311 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:52,313 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091777a717815546db95c32600156be55c_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091777a717815546db95c32600156be55c_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:52,314 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/03bf27b2815b4621bad4d92938a4915a, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:52,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/03bf27b2815b4621bad4d92938a4915a is 175, key is test_row_0/A:col10/1733732211893/Put/seqid=0 2024-12-09T08:16:52,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742387_1563 (size=39599) 2024-12-09T08:16:52,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T08:16:52,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:52,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:52,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:52,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732272531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732272531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732272541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T08:16:52,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:52,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:52,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:52,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732272541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:52,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732272541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T08:16:52,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T08:16:52,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:52,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:52,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:52,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,718 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/03bf27b2815b4621bad4d92938a4915a 2024-12-09T08:16:52,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/04d5f844a69e444baf48ffda7fd192f8 is 50, key is test_row_0/B:col10/1733732211893/Put/seqid=0 2024-12-09T08:16:52,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742388_1564 (size=12051) 2024-12-09T08:16:52,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/04d5f844a69e444baf48ffda7fd192f8 2024-12-09T08:16:52,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/2365d2f22cfa4102b987ac3de0807671 is 50, key is test_row_0/C:col10/1733732211893/Put/seqid=0 2024-12-09T08:16:52,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742389_1565 (size=12051) 2024-12-09T08:16:52,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T08:16:52,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:52,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:52,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:52,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:53,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T08:16:53,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:53,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:53,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:53,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:53,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:53,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732273041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:53,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732273041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:53,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732273051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:53,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732273051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:53,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732273051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T08:16:53,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/2365d2f22cfa4102b987ac3de0807671 2024-12-09T08:16:53,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/03bf27b2815b4621bad4d92938a4915a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/03bf27b2815b4621bad4d92938a4915a 2024-12-09T08:16:53,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/03bf27b2815b4621bad4d92938a4915a, entries=200, sequenceid=129, filesize=38.7 K 2024-12-09T08:16:53,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/04d5f844a69e444baf48ffda7fd192f8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/04d5f844a69e444baf48ffda7fd192f8 2024-12-09T08:16:53,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/04d5f844a69e444baf48ffda7fd192f8, entries=150, sequenceid=129, filesize=11.8 K 2024-12-09T08:16:53,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/2365d2f22cfa4102b987ac3de0807671 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/2365d2f22cfa4102b987ac3de0807671 2024-12-09T08:16:53,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T08:16:53,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:53,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:53,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:53,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:53,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:53,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/2365d2f22cfa4102b987ac3de0807671, entries=150, sequenceid=129, filesize=11.8 K 2024-12-09T08:16:53,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 19c9630e7ab12af49561314a634ad52e in 1262ms, sequenceid=129, compaction requested=true 2024-12-09T08:16:53,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:53,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:53,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:53,157 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:53,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:53,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:53,157 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:53,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:53,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:53,157 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:53,157 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101715 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:53,157 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/B is initiating minor compaction (all files) 2024-12-09T08:16:53,157 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/A is initiating minor compaction (all files) 2024-12-09T08:16:53,158 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/A in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:53,158 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/B in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:53,158 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/74a659e4520d4e1596aa019edf38b18c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/49c02ae75e314c47a7e9e4ffb5ec9485, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/04d5f844a69e444baf48ffda7fd192f8] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=35.4 K 2024-12-09T08:16:53,158 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/2ad158fac58e435e8f28a90c4df541c5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/c7b45edc7e2c4ddb9d80e416f600f0b9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/03bf27b2815b4621bad4d92938a4915a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=99.3 K 2024-12-09T08:16:53,158 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:53,158 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/2ad158fac58e435e8f28a90c4df541c5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/c7b45edc7e2c4ddb9d80e416f600f0b9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/03bf27b2815b4621bad4d92938a4915a] 2024-12-09T08:16:53,158 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 74a659e4520d4e1596aa019edf38b18c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733732207445 2024-12-09T08:16:53,158 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ad158fac58e435e8f28a90c4df541c5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733732207445 2024-12-09T08:16:53,159 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 49c02ae75e314c47a7e9e4ffb5ec9485, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733732207567 2024-12-09T08:16:53,159 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7b45edc7e2c4ddb9d80e416f600f0b9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733732207567 2024-12-09T08:16:53,159 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d5f844a69e444baf48ffda7fd192f8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733732211893 2024-12-09T08:16:53,159 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03bf27b2815b4621bad4d92938a4915a, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733732211893 2024-12-09T08:16:53,164 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:53,166 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#B#compaction#483 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:53,166 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209969b897bddb5467fac5adfe7036cf575_19c9630e7ab12af49561314a634ad52e store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:53,166 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/217154932ee84bd999926c5c67b1b694 is 50, key is test_row_0/B:col10/1733732211893/Put/seqid=0 2024-12-09T08:16:53,167 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209969b897bddb5467fac5adfe7036cf575_19c9630e7ab12af49561314a634ad52e, store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:53,167 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209969b897bddb5467fac5adfe7036cf575_19c9630e7ab12af49561314a634ad52e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:53,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742391_1567 (size=4469) 2024-12-09T08:16:53,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742390_1566 (size=12359) 2024-12-09T08:16:53,189 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/217154932ee84bd999926c5c67b1b694 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/217154932ee84bd999926c5c67b1b694 2024-12-09T08:16:53,193 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/B of 19c9630e7ab12af49561314a634ad52e into 217154932ee84bd999926c5c67b1b694(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:53,193 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:53,193 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/B, priority=13, startTime=1733732213157; duration=0sec 2024-12-09T08:16:53,193 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:53,193 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:B 2024-12-09T08:16:53,193 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:53,194 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:53,194 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/C is initiating minor compaction (all files) 2024-12-09T08:16:53,194 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/C in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:53,194 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/db64cac4a61e48ccaea3ad951832c048, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/30254500f5674510835ba7c43ae62866, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/2365d2f22cfa4102b987ac3de0807671] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=35.4 K 2024-12-09T08:16:53,195 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting db64cac4a61e48ccaea3ad951832c048, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733732207445 2024-12-09T08:16:53,195 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 30254500f5674510835ba7c43ae62866, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733732207567 2024-12-09T08:16:53,196 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 2365d2f22cfa4102b987ac3de0807671, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733732211893 2024-12-09T08:16:53,201 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#C#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:53,201 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/ee3dc16c573349ecae707b5436f3f2a8 is 50, key is test_row_0/C:col10/1733732211893/Put/seqid=0 2024-12-09T08:16:53,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742392_1568 (size=12359) 2024-12-09T08:16:53,208 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/ee3dc16c573349ecae707b5436f3f2a8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ee3dc16c573349ecae707b5436f3f2a8 2024-12-09T08:16:53,212 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/C of 19c9630e7ab12af49561314a634ad52e into ee3dc16c573349ecae707b5436f3f2a8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:53,212 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:53,212 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/C, priority=13, startTime=1733732213157; duration=0sec 2024-12-09T08:16:53,212 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:53,212 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:C 2024-12-09T08:16:53,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T08:16:53,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:53,306 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T08:16:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:53,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098392c76ea0a04694a486a8f51360a5d1_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732211905/Put/seqid=0 2024-12-09T08:16:53,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742393_1569 (size=12304) 2024-12-09T08:16:53,583 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#A#compaction#482 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:53,584 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/0cc1df8ebb7d4bfc8785bf85dbef9fa7 is 175, key is test_row_0/A:col10/1733732211893/Put/seqid=0 2024-12-09T08:16:53,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742394_1570 (size=31313) 2024-12-09T08:16:53,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:53,721 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098392c76ea0a04694a486a8f51360a5d1_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098392c76ea0a04694a486a8f51360a5d1_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:53,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/b1d0158d520243c0adbcf65af04844dd, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:53,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/b1d0158d520243c0adbcf65af04844dd is 175, key is test_row_0/A:col10/1733732211905/Put/seqid=0 2024-12-09T08:16:53,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742395_1571 (size=31105) 2024-12-09T08:16:53,725 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/b1d0158d520243c0adbcf65af04844dd 2024-12-09T08:16:53,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/0b4fac49e73c4fe9b3fe8dca8b3abf03 is 50, key is test_row_0/B:col10/1733732211905/Put/seqid=0 2024-12-09T08:16:53,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742396_1572 (size=12151) 2024-12-09T08:16:53,999 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/0cc1df8ebb7d4bfc8785bf85dbef9fa7 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/0cc1df8ebb7d4bfc8785bf85dbef9fa7 2024-12-09T08:16:54,002 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/A of 19c9630e7ab12af49561314a634ad52e into 0cc1df8ebb7d4bfc8785bf85dbef9fa7(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:54,002 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:54,002 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/A, priority=13, startTime=1733732213157; duration=0sec 2024-12-09T08:16:54,002 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:54,002 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:A 2024-12-09T08:16:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:54,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:54,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732274069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732274069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732274070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732214064, max timestamp=1733732214073Num uniques 859; 2024-12-09T08:16:54,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732214064, max timestamp=1733732214073Num uniques 859; 2024-12-09T08:16:54,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:16:54,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:16:54,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733732214064, max timestamp=1733732214073Num uniques 859; 2024-12-09T08:16:54,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T08:16:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T08:16:54,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732274116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732274118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,134 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/0b4fac49e73c4fe9b3fe8dca8b3abf03 2024-12-09T08:16:54,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/a3f5624ac0eb41b48a7e638172a316ab is 50, key is test_row_0/C:col10/1733732211905/Put/seqid=0 2024-12-09T08:16:54,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742397_1573 (size=12151) 2024-12-09T08:16:54,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732274181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732274181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732274181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732274231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732274231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732274391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732274393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732274393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732274441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732274441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,543 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/a3f5624ac0eb41b48a7e638172a316ab 2024-12-09T08:16:54,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/b1d0158d520243c0adbcf65af04844dd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/b1d0158d520243c0adbcf65af04844dd 2024-12-09T08:16:54,549 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/b1d0158d520243c0adbcf65af04844dd, entries=150, sequenceid=143, filesize=30.4 K 2024-12-09T08:16:54,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/0b4fac49e73c4fe9b3fe8dca8b3abf03 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/0b4fac49e73c4fe9b3fe8dca8b3abf03 2024-12-09T08:16:54,553 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/0b4fac49e73c4fe9b3fe8dca8b3abf03, entries=150, sequenceid=143, filesize=11.9 K 2024-12-09T08:16:54,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/a3f5624ac0eb41b48a7e638172a316ab as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/a3f5624ac0eb41b48a7e638172a316ab 2024-12-09T08:16:54,556 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/a3f5624ac0eb41b48a7e638172a316ab, entries=150, sequenceid=143, filesize=11.9 K 2024-12-09T08:16:54,557 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 19c9630e7ab12af49561314a634ad52e in 1252ms, sequenceid=143, compaction requested=false 2024-12-09T08:16:54,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:54,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:54,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-09T08:16:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-09T08:16:54,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-09T08:16:54,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6240 sec 2024-12-09T08:16:54,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.6320 sec 2024-12-09T08:16:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:54,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-09T08:16:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:54,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096526fde6838344ea856be771631555df_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732214114/Put/seqid=0 2024-12-09T08:16:54,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732274711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742398_1574 (size=12304) 2024-12-09T08:16:54,716 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:54,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732274714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732274715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,718 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096526fde6838344ea856be771631555df_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096526fde6838344ea856be771631555df_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,719 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/857df265ffde4cd19b8eae474b606498, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:54,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/857df265ffde4cd19b8eae474b606498 is 175, key is test_row_0/A:col10/1733732214114/Put/seqid=0 2024-12-09T08:16:54,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742399_1575 (size=31105) 2024-12-09T08:16:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732274751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732274751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732274821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732274821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732274821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:55,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732275031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732275031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:55,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732275031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:55,123 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/857df265ffde4cd19b8eae474b606498 2024-12-09T08:16:55,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/6f7b680684b544bb84a26b71b94df2bf is 50, key is test_row_0/B:col10/1733732214114/Put/seqid=0 2024-12-09T08:16:55,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742400_1576 (size=12151) 2024-12-09T08:16:55,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732275261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:55,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732275261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:55,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732275341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732275341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:55,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732275341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:55,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/6f7b680684b544bb84a26b71b94df2bf 2024-12-09T08:16:55,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/829e9ff40e14424cbb9c1f818bc6d83a is 50, key is test_row_0/C:col10/1733732214114/Put/seqid=0 2024-12-09T08:16:55,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742401_1577 (size=12151) 2024-12-09T08:16:55,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732275851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:55,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:55,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732275851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:55,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:55,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732275852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:55,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/829e9ff40e14424cbb9c1f818bc6d83a 2024-12-09T08:16:55,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/857df265ffde4cd19b8eae474b606498 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/857df265ffde4cd19b8eae474b606498 2024-12-09T08:16:55,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/857df265ffde4cd19b8eae474b606498, entries=150, sequenceid=180, filesize=30.4 K 2024-12-09T08:16:55,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/6f7b680684b544bb84a26b71b94df2bf as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/6f7b680684b544bb84a26b71b94df2bf 2024-12-09T08:16:55,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/6f7b680684b544bb84a26b71b94df2bf, entries=150, sequenceid=180, filesize=11.9 K 2024-12-09T08:16:55,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/829e9ff40e14424cbb9c1f818bc6d83a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/829e9ff40e14424cbb9c1f818bc6d83a 2024-12-09T08:16:55,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/829e9ff40e14424cbb9c1f818bc6d83a, entries=150, sequenceid=180, filesize=11.9 K 2024-12-09T08:16:55,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 19c9630e7ab12af49561314a634ad52e in 1268ms, sequenceid=180, compaction requested=true 2024-12-09T08:16:55,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:55,971 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:55,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:16:55,973 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93523 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:55,973 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/A is initiating minor compaction (all files) 2024-12-09T08:16:55,973 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/A in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:55,974 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/0cc1df8ebb7d4bfc8785bf85dbef9fa7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/b1d0158d520243c0adbcf65af04844dd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/857df265ffde4cd19b8eae474b606498] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=91.3 K 2024-12-09T08:16:55,974 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:55,974 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/0cc1df8ebb7d4bfc8785bf85dbef9fa7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/b1d0158d520243c0adbcf65af04844dd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/857df265ffde4cd19b8eae474b606498] 2024-12-09T08:16:55,974 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cc1df8ebb7d4bfc8785bf85dbef9fa7, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733732211893 2024-12-09T08:16:55,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:55,975 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1d0158d520243c0adbcf65af04844dd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733732211900 2024-12-09T08:16:55,975 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:55,975 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 857df265ffde4cd19b8eae474b606498, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733732214113 2024-12-09T08:16:55,977 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:55,977 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/B is initiating minor compaction (all files) 2024-12-09T08:16:55,977 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/B in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:55,978 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/217154932ee84bd999926c5c67b1b694, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/0b4fac49e73c4fe9b3fe8dca8b3abf03, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/6f7b680684b544bb84a26b71b94df2bf] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=35.8 K 2024-12-09T08:16:55,979 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 217154932ee84bd999926c5c67b1b694, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733732211893 2024-12-09T08:16:55,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:16:55,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:55,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:16:55,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:55,981 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b4fac49e73c4fe9b3fe8dca8b3abf03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733732211900 2024-12-09T08:16:55,981 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f7b680684b544bb84a26b71b94df2bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733732214113 2024-12-09T08:16:55,997 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:56,011 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:56,011 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/ebfbe3dadb5a433294405c4e33681875 is 50, key is test_row_0/B:col10/1733732214114/Put/seqid=0 2024-12-09T08:16:56,017 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412094e85f008290940f5b38d521d5ceb73b1_19c9630e7ab12af49561314a634ad52e store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:56,020 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412094e85f008290940f5b38d521d5ceb73b1_19c9630e7ab12af49561314a634ad52e, store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:56,020 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094e85f008290940f5b38d521d5ceb73b1_19c9630e7ab12af49561314a634ad52e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:56,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742402_1578 (size=12561) 2024-12-09T08:16:56,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742403_1579 (size=4469) 2024-12-09T08:16:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T08:16:56,101 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:56,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-09T08:16:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T08:16:56,107 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:56,108 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:56,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T08:16:56,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T08:16:56,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:56,261 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T08:16:56,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:56,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:56,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:56,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:56,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:56,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:56,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120927d0368c4a2c4b85baf9eb667964839b_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732214713/Put/seqid=0 2024-12-09T08:16:56,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742404_1580 (size=12304) 2024-12-09T08:16:56,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:56,331 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120927d0368c4a2c4b85baf9eb667964839b_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120927d0368c4a2c4b85baf9eb667964839b_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:56,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/34e576901d3b4ba8a9c7a08e45fefa2b, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/34e576901d3b4ba8a9c7a08e45fefa2b is 175, key is test_row_0/A:col10/1733732214713/Put/seqid=0 2024-12-09T08:16:56,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732216320, max timestamp=1733732216335Num uniques 859; 2024-12-09T08:16:56,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732216320, max timestamp=1733732216335Num uniques 859; 2024-12-09T08:16:56,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:16:56,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:16:56,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732216320, max timestamp=1733732216335Num uniques 859; 2024-12-09T08:16:56,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:16:56,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742405_1581 (size=31105) 2024-12-09T08:16:56,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732276379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:56,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732276381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:56,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T08:16:56,460 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/ebfbe3dadb5a433294405c4e33681875 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/ebfbe3dadb5a433294405c4e33681875 2024-12-09T08:16:56,470 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/B of 19c9630e7ab12af49561314a634ad52e into ebfbe3dadb5a433294405c4e33681875(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:56,470 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:56,470 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/B, priority=13, startTime=1733732215975; duration=0sec 2024-12-09T08:16:56,470 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:16:56,470 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:B 2024-12-09T08:16:56,470 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:16:56,471 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:16:56,471 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/C is initiating minor compaction (all files) 2024-12-09T08:16:56,471 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/C in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:56,471 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ee3dc16c573349ecae707b5436f3f2a8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/a3f5624ac0eb41b48a7e638172a316ab, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/829e9ff40e14424cbb9c1f818bc6d83a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=35.8 K 2024-12-09T08:16:56,472 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3dc16c573349ecae707b5436f3f2a8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733732211893 2024-12-09T08:16:56,472 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting a3f5624ac0eb41b48a7e638172a316ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733732211900 2024-12-09T08:16:56,472 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 829e9ff40e14424cbb9c1f818bc6d83a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733732214113 2024-12-09T08:16:56,487 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#A#compaction#491 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:56,487 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/25df820ae3f9433085ace54f243b3c6d is 175, key is test_row_0/A:col10/1733732214114/Put/seqid=0 2024-12-09T08:16:56,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732276491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:56,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732276491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:56,498 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:16:56,499 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/5793c0ce42474efd9756a87ab6f32dfd is 50, key is test_row_0/C:col10/1733732214114/Put/seqid=0 2024-12-09T08:16:56,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742406_1582 (size=31515) 2024-12-09T08:16:56,508 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/25df820ae3f9433085ace54f243b3c6d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/25df820ae3f9433085ace54f243b3c6d 2024-12-09T08:16:56,511 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/A of 19c9630e7ab12af49561314a634ad52e into 25df820ae3f9433085ace54f243b3c6d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:56,512 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:56,512 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/A, priority=13, startTime=1733732215971; duration=0sec 2024-12-09T08:16:56,512 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:56,512 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:A 2024-12-09T08:16:56,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742407_1583 (size=12561) 2024-12-09T08:16:56,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732276701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:56,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732276701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T08:16:56,746 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/34e576901d3b4ba8a9c7a08e45fefa2b 2024-12-09T08:16:56,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/ceff3cf38bf84fa2b6acab612bd6f925 is 50, key is test_row_0/B:col10/1733732214713/Put/seqid=0 2024-12-09T08:16:56,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742408_1584 (size=12151) 2024-12-09T08:16:56,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732276861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:56,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732276862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:56,863 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 19c9630e7ab12af49561314a634ad52e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:56,864 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 19c9630e7ab12af49561314a634ad52e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:56,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732276871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:56,873 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 19c9630e7ab12af49561314a634ad52e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:56,919 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/5793c0ce42474efd9756a87ab6f32dfd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/5793c0ce42474efd9756a87ab6f32dfd 2024-12-09T08:16:56,922 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/C of 19c9630e7ab12af49561314a634ad52e into 5793c0ce42474efd9756a87ab6f32dfd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:16:56,923 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:56,923 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/C, priority=13, startTime=1733732215979; duration=0sec 2024-12-09T08:16:56,923 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:16:56,923 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:C 2024-12-09T08:16:57,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732277011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732277011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:57,155 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/ceff3cf38bf84fa2b6acab612bd6f925 2024-12-09T08:16:57,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/cc94b8e67937485b98062a26b4add137 is 50, key is test_row_0/C:col10/1733732214713/Put/seqid=0 2024-12-09T08:16:57,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742409_1585 (size=12151) 2024-12-09T08:16:57,175 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/cc94b8e67937485b98062a26b4add137 2024-12-09T08:16:57,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/34e576901d3b4ba8a9c7a08e45fefa2b as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/34e576901d3b4ba8a9c7a08e45fefa2b 2024-12-09T08:16:57,181 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/34e576901d3b4ba8a9c7a08e45fefa2b, entries=150, sequenceid=191, filesize=30.4 K 2024-12-09T08:16:57,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/ceff3cf38bf84fa2b6acab612bd6f925 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/ceff3cf38bf84fa2b6acab612bd6f925 2024-12-09T08:16:57,185 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/ceff3cf38bf84fa2b6acab612bd6f925, entries=150, sequenceid=191, filesize=11.9 K 2024-12-09T08:16:57,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/cc94b8e67937485b98062a26b4add137 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/cc94b8e67937485b98062a26b4add137 2024-12-09T08:16:57,191 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/cc94b8e67937485b98062a26b4add137, entries=150, sequenceid=191, filesize=11.9 K 2024-12-09T08:16:57,191 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 19c9630e7ab12af49561314a634ad52e in 930ms, sequenceid=191, compaction requested=false 2024-12-09T08:16:57,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:57,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:57,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-09T08:16:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-09T08:16:57,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-09T08:16:57,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0850 sec 2024-12-09T08:16:57,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.0930 sec 2024-12-09T08:16:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T08:16:57,241 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:57,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-09T08:16:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T08:16:57,243 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:57,244 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:57,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T08:16:57,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T08:16:57,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:57,396 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T08:16:57,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:57,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:57,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:57,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:57,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:57,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:16:57,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099eac5a3fb21d472697053dd0f296f18c_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732216377/Put/seqid=0 2024-12-09T08:16:57,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742410_1586 (size=12304) 2024-12-09T08:16:57,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:57,412 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099eac5a3fb21d472697053dd0f296f18c_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099eac5a3fb21d472697053dd0f296f18c_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:57,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/fc05f73ceeb8492e8e57fa72d6cbc59c, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:57,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/fc05f73ceeb8492e8e57fa72d6cbc59c is 175, key is test_row_0/A:col10/1733732216377/Put/seqid=0 2024-12-09T08:16:57,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742411_1587 (size=31105) 2024-12-09T08:16:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:57,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:57,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732277532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:57,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:57,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732277535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T08:16:57,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732277641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:57,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732277641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:57,816 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/fc05f73ceeb8492e8e57fa72d6cbc59c 2024-12-09T08:16:57,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/e0176dce0a414276acac1f93e6e12304 is 50, key is test_row_0/B:col10/1733732216377/Put/seqid=0 2024-12-09T08:16:57,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742412_1588 (size=12151) 2024-12-09T08:16:57,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732277851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732277851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T08:16:58,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:58,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732278161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:58,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:58,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:58,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732278161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:58,231 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/e0176dce0a414276acac1f93e6e12304 2024-12-09T08:16:58,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/ef0d9294386d45a8b32eeb68712aa724 is 50, key is test_row_0/C:col10/1733732216377/Put/seqid=0 2024-12-09T08:16:58,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742413_1589 (size=12151) 2024-12-09T08:16:58,239 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/ef0d9294386d45a8b32eeb68712aa724 2024-12-09T08:16:58,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/fc05f73ceeb8492e8e57fa72d6cbc59c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/fc05f73ceeb8492e8e57fa72d6cbc59c 2024-12-09T08:16:58,246 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/fc05f73ceeb8492e8e57fa72d6cbc59c, entries=150, sequenceid=230, filesize=30.4 K 2024-12-09T08:16:58,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/e0176dce0a414276acac1f93e6e12304 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/e0176dce0a414276acac1f93e6e12304 2024-12-09T08:16:58,252 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/e0176dce0a414276acac1f93e6e12304, entries=150, sequenceid=230, filesize=11.9 K 2024-12-09T08:16:58,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/ef0d9294386d45a8b32eeb68712aa724 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ef0d9294386d45a8b32eeb68712aa724 2024-12-09T08:16:58,256 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ef0d9294386d45a8b32eeb68712aa724, entries=150, sequenceid=230, filesize=11.9 K 2024-12-09T08:16:58,257 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 19c9630e7ab12af49561314a634ad52e in 861ms, sequenceid=230, compaction requested=true 2024-12-09T08:16:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-09T08:16:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-09T08:16:58,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-09T08:16:58,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-12-09T08:16:58,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0190 sec 2024-12-09T08:16:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T08:16:58,382 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:58,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-09T08:16:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T08:16:58,383 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:58,384 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:58,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T08:16:58,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-09T08:16:58,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:58,536 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:16:58,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:58,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:58,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:58,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:58,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:58,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:58,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098b5beed2efa241b3b8fa5873766a79e9_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732217534/Put/seqid=0 2024-12-09T08:16:58,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742414_1590 (size=12304) 2024-12-09T08:16:58,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,589 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098b5beed2efa241b3b8fa5873766a79e9_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098b5beed2efa241b3b8fa5873766a79e9_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:58,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/ad7b94129d594d879c8f2b95d54b2a7c, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/ad7b94129d594d879c8f2b95d54b2a7c is 175, key is test_row_0/A:col10/1733732217534/Put/seqid=0 2024-12-09T08:16:58,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742415_1591 (size=31105) 2024-12-09T08:16:58,594 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/ad7b94129d594d879c8f2b95d54b2a7c 2024-12-09T08:16:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/718d39c876184204ae3d5fb8d98e8abf is 50, key is test_row_0/B:col10/1733732217534/Put/seqid=0 2024-12-09T08:16:58,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742416_1592 (size=12151) 2024-12-09T08:16:58,611 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/718d39c876184204ae3d5fb8d98e8abf 2024-12-09T08:16:58,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/ac91a13972fb4673a5d61fe1acc2b68a is 50, key is test_row_0/C:col10/1733732217534/Put/seqid=0 2024-12-09T08:16:58,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742417_1593 (size=12151) 2024-12-09T08:16:58,639 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/ac91a13972fb4673a5d61fe1acc2b68a 2024-12-09T08:16:58,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/ad7b94129d594d879c8f2b95d54b2a7c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/ad7b94129d594d879c8f2b95d54b2a7c 2024-12-09T08:16:58,652 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/ad7b94129d594d879c8f2b95d54b2a7c, entries=150, sequenceid=240, filesize=30.4 K 2024-12-09T08:16:58,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/718d39c876184204ae3d5fb8d98e8abf as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/718d39c876184204ae3d5fb8d98e8abf 2024-12-09T08:16:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,657 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/718d39c876184204ae3d5fb8d98e8abf, entries=150, sequenceid=240, filesize=11.9 K 2024-12-09T08:16:58,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/ac91a13972fb4673a5d61fe1acc2b68a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ac91a13972fb4673a5d61fe1acc2b68a 2024-12-09T08:16:58,662 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ac91a13972fb4673a5d61fe1acc2b68a, entries=150, sequenceid=240, filesize=11.9 K 2024-12-09T08:16:58,663 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 19c9630e7ab12af49561314a634ad52e in 127ms, sequenceid=240, compaction requested=true 2024-12-09T08:16:58,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:16:58,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:58,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-09T08:16:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-09T08:16:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-09T08:16:58,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 281 msec 2024-12-09T08:16:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 287 msec 2024-12-09T08:16:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T08:16:58,702 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:16:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:16:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-09T08:16:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T08:16:58,704 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:16:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,709 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:16:58,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:16:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:58,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:16:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:16:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:16:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:16:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:16:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e6198f1967d345f99df1ee01c9ce91b5_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732218750/Put/seqid=0 2024-12-09T08:16:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:58,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742418_1594 (size=19774) 2024-12-09T08:16:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T08:16:58,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732218795, max timestamp=1733732218813Num uniques 859; 2024-12-09T08:16:58,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732218795, max timestamp=1733732218813Num uniques 859; 2024-12-09T08:16:58,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:16:58,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:16:58,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732218795, max timestamp=1733732218813Num uniques 859; 2024-12-09T08:16:58,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:16:58,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732278854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:58,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T08:16:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:58,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:58,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732278859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:58,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:58,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732278881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:58,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732278881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:58,886 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 19c9630e7ab12af49561314a634ad52e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:58,886 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 19c9630e7ab12af49561314a634ad52e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:58,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732278891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:58,895 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 19c9630e7ab12af49561314a634ad52e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:16:58,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:58,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732278971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:58,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:58,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732278971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:58,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:58,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:59,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T08:16:59,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T08:16:59,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T08:16:59,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:59,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732279181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:59,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732279181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:59,198 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:16:59,201 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e6198f1967d345f99df1ee01c9ce91b5_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e6198f1967d345f99df1ee01c9ce91b5_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:16:59,201 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/6e69ab2956674208928a846c8a4d85d4, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:16:59,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/6e69ab2956674208928a846c8a4d85d4 is 175, key is test_row_0/A:col10/1733732218750/Put/seqid=0 2024-12-09T08:16:59,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742419_1595 (size=57033) 2024-12-09T08:16:59,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T08:16:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:59,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T08:16:59,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T08:16:59,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:59,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732279491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:59,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:16:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732279491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:16:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:16:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:16:59,612 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/6e69ab2956674208928a846c8a4d85d4 2024-12-09T08:16:59,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/01e9b972dfcf4c5892c58268c91ea454 is 50, key is test_row_0/B:col10/1733732218750/Put/seqid=0 2024-12-09T08:16:59,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742420_1596 (size=12151) 2024-12-09T08:16:59,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T08:16:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T08:16:59,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:59,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T08:16:59,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T08:16:59,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:16:59,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:16:59,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:16:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:00,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732280001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:00,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732280001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:00,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:00,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:00,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:00,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:00,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/01e9b972dfcf4c5892c58268c91ea454 2024-12-09T08:17:00,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/3504b125f26f45799d068ef78926b33e is 50, key is test_row_0/C:col10/1733732218750/Put/seqid=0 2024-12-09T08:17:00,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742421_1597 (size=12151) 2024-12-09T08:17:00,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/3504b125f26f45799d068ef78926b33e 2024-12-09T08:17:00,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/6e69ab2956674208928a846c8a4d85d4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/6e69ab2956674208928a846c8a4d85d4 2024-12-09T08:17:00,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/6e69ab2956674208928a846c8a4d85d4, entries=300, sequenceid=250, filesize=55.7 K 2024-12-09T08:17:00,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/01e9b972dfcf4c5892c58268c91ea454 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/01e9b972dfcf4c5892c58268c91ea454 2024-12-09T08:17:00,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/01e9b972dfcf4c5892c58268c91ea454, entries=150, sequenceid=250, filesize=11.9 K 2024-12-09T08:17:00,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/3504b125f26f45799d068ef78926b33e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/3504b125f26f45799d068ef78926b33e 2024-12-09T08:17:00,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/3504b125f26f45799d068ef78926b33e, entries=150, sequenceid=250, filesize=11.9 K 2024-12-09T08:17:00,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 19c9630e7ab12af49561314a634ad52e in 1324ms, sequenceid=250, compaction requested=true 2024-12-09T08:17:00,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:17:00,076 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T08:17:00,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:17:00,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:17:00,077 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T08:17:00,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:17:00,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:17:00,077 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 181863 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T08:17:00,077 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/A is initiating minor compaction (all files) 2024-12-09T08:17:00,078 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/A in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:00,078 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/25df820ae3f9433085ace54f243b3c6d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/34e576901d3b4ba8a9c7a08e45fefa2b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/fc05f73ceeb8492e8e57fa72d6cbc59c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/ad7b94129d594d879c8f2b95d54b2a7c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/6e69ab2956674208928a846c8a4d85d4] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=177.6 K 2024-12-09T08:17:00,078 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:00,078 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/25df820ae3f9433085ace54f243b3c6d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/34e576901d3b4ba8a9c7a08e45fefa2b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/fc05f73ceeb8492e8e57fa72d6cbc59c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/ad7b94129d594d879c8f2b95d54b2a7c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/6e69ab2956674208928a846c8a4d85d4] 2024-12-09T08:17:00,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:17:00,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:17:00,079 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T08:17:00,079 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25df820ae3f9433085ace54f243b3c6d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733732214113 2024-12-09T08:17:00,080 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/B is initiating minor compaction (all files) 2024-12-09T08:17:00,080 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/B in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:00,080 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/ebfbe3dadb5a433294405c4e33681875, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/ceff3cf38bf84fa2b6acab612bd6f925, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/e0176dce0a414276acac1f93e6e12304, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/718d39c876184204ae3d5fb8d98e8abf, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/01e9b972dfcf4c5892c58268c91ea454] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=59.7 K 2024-12-09T08:17:00,080 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34e576901d3b4ba8a9c7a08e45fefa2b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733732214710 2024-12-09T08:17:00,080 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ebfbe3dadb5a433294405c4e33681875, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733732214113 2024-12-09T08:17:00,080 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ceff3cf38bf84fa2b6acab612bd6f925, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733732214710 2024-12-09T08:17:00,080 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc05f73ceeb8492e8e57fa72d6cbc59c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733732216369 2024-12-09T08:17:00,081 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting e0176dce0a414276acac1f93e6e12304, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733732216369 2024-12-09T08:17:00,081 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad7b94129d594d879c8f2b95d54b2a7c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733732217523 2024-12-09T08:17:00,081 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 718d39c876184204ae3d5fb8d98e8abf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733732217523 2024-12-09T08:17:00,081 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e69ab2956674208928a846c8a4d85d4, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733732218716 2024-12-09T08:17:00,081 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 01e9b972dfcf4c5892c58268c91ea454, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733732218716 2024-12-09T08:17:00,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T08:17:00,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:00,082 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T08:17:00,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:17:00,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:17:00,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:17:00,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:17:00,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:17:00,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:17:00,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a53d1d9a6b054ef98f3414823088e258_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732218858/Put/seqid=0 2024-12-09T08:17:00,123 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#B#compaction#507 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:17:00,123 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/12a338d496f74c8d99e7d96fb70358da is 50, key is test_row_0/B:col10/1733732218750/Put/seqid=0 2024-12-09T08:17:00,124 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:00,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742423_1599 (size=12731) 2024-12-09T08:17:00,141 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/12a338d496f74c8d99e7d96fb70358da as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/12a338d496f74c8d99e7d96fb70358da 2024-12-09T08:17:00,146 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/B of 19c9630e7ab12af49561314a634ad52e into 12a338d496f74c8d99e7d96fb70358da(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:17:00,146 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:17:00,146 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/B, priority=11, startTime=1733732220076; duration=0sec 2024-12-09T08:17:00,146 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:17:00,146 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:B 2024-12-09T08:17:00,146 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T08:17:00,147 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T08:17:00,147 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/C is initiating minor compaction (all files) 2024-12-09T08:17:00,147 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/C in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:00,148 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/5793c0ce42474efd9756a87ab6f32dfd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/cc94b8e67937485b98062a26b4add137, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ef0d9294386d45a8b32eeb68712aa724, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ac91a13972fb4673a5d61fe1acc2b68a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/3504b125f26f45799d068ef78926b33e] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=59.7 K 2024-12-09T08:17:00,149 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 5793c0ce42474efd9756a87ab6f32dfd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733732214113 2024-12-09T08:17:00,150 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting cc94b8e67937485b98062a26b4add137, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733732214710 2024-12-09T08:17:00,150 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ef0d9294386d45a8b32eeb68712aa724, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733732216369 2024-12-09T08:17:00,150 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting ac91a13972fb4673a5d61fe1acc2b68a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733732217523 2024-12-09T08:17:00,150 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3504b125f26f45799d068ef78926b33e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733732218716 2024-12-09T08:17:00,153 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209388e52c7bc834328b0fc70754fd1b5dd_19c9630e7ab12af49561314a634ad52e store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:00,156 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209388e52c7bc834328b0fc70754fd1b5dd_19c9630e7ab12af49561314a634ad52e, store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:00,156 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209388e52c7bc834328b0fc70754fd1b5dd_19c9630e7ab12af49561314a634ad52e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:00,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742422_1598 (size=12454) 2024-12-09T08:17:00,175 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#C#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:17:00,176 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/3c08721501df4d608ecb7f3b100f134c is 50, key is test_row_0/C:col10/1733732218750/Put/seqid=0 2024-12-09T08:17:00,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742424_1600 (size=4469) 2024-12-09T08:17:00,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742425_1601 (size=12731) 2024-12-09T08:17:00,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:00,565 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a53d1d9a6b054ef98f3414823088e258_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a53d1d9a6b054ef98f3414823088e258_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:00,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/80a52378731b47f09c93bd2996e28f24, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:00,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/80a52378731b47f09c93bd2996e28f24 is 175, key is test_row_0/A:col10/1733732218858/Put/seqid=0 2024-12-09T08:17:00,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742426_1602 (size=31255) 2024-12-09T08:17:00,578 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#A#compaction#508 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:17:00,579 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/55199f07c0464d6cba6fb120a087bd08 is 175, key is test_row_0/A:col10/1733732218750/Put/seqid=0 2024-12-09T08:17:00,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742427_1603 (size=31685) 2024-12-09T08:17:00,607 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/3c08721501df4d608ecb7f3b100f134c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/3c08721501df4d608ecb7f3b100f134c 2024-12-09T08:17:00,611 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/C of 19c9630e7ab12af49561314a634ad52e into 3c08721501df4d608ecb7f3b100f134c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:17:00,611 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:17:00,611 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/C, priority=11, startTime=1733732220077; duration=0sec 2024-12-09T08:17:00,611 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:17:00,611 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:C 2024-12-09T08:17:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T08:17:00,971 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/80a52378731b47f09c93bd2996e28f24 2024-12-09T08:17:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/1321fb4121434f5da70ff0589ad933b0 is 50, key is test_row_0/B:col10/1733732218858/Put/seqid=0 2024-12-09T08:17:00,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742428_1604 (size=12301) 2024-12-09T08:17:00,984 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/1321fb4121434f5da70ff0589ad933b0 2024-12-09T08:17:00,988 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/55199f07c0464d6cba6fb120a087bd08 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/55199f07c0464d6cba6fb120a087bd08 2024-12-09T08:17:00,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/1a117d1e3eb44d1892d69fddf893d435 is 50, key is test_row_0/C:col10/1733732218858/Put/seqid=0 2024-12-09T08:17:00,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742429_1605 (size=12301) 2024-12-09T08:17:00,994 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/1a117d1e3eb44d1892d69fddf893d435 2024-12-09T08:17:00,995 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/A of 19c9630e7ab12af49561314a634ad52e into 55199f07c0464d6cba6fb120a087bd08(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:17:00,995 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:17:00,995 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/A, priority=11, startTime=1733732220076; duration=0sec 2024-12-09T08:17:00,996 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:17:00,996 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:A 2024-12-09T08:17:00,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/80a52378731b47f09c93bd2996e28f24 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/80a52378731b47f09c93bd2996e28f24 2024-12-09T08:17:01,005 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/80a52378731b47f09c93bd2996e28f24, entries=150, sequenceid=286, filesize=30.5 K 2024-12-09T08:17:01,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/1321fb4121434f5da70ff0589ad933b0 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/1321fb4121434f5da70ff0589ad933b0 2024-12-09T08:17:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,010 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/1321fb4121434f5da70ff0589ad933b0, entries=150, sequenceid=286, filesize=12.0 K 2024-12-09T08:17:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/1a117d1e3eb44d1892d69fddf893d435 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/1a117d1e3eb44d1892d69fddf893d435 2024-12-09T08:17:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,014 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/1a117d1e3eb44d1892d69fddf893d435, entries=150, sequenceid=286, filesize=12.0 K 2024-12-09T08:17:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,015 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for 19c9630e7ab12af49561314a634ad52e in 933ms, sequenceid=286, compaction requested=false 2024-12-09T08:17:01,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:17:01,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:01,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-09T08:17:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-09T08:17:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-09T08:17:01,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3070 sec 2024-12-09T08:17:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.3160 sec 2024-12-09T08:17:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:01,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:17:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:17:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:17:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:17:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:17:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:17:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:17:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ef14ba692d66491d959e5f9234d0e1b5_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732221068/Put/seqid=0 2024-12-09T08:17:01,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742430_1606 (size=22618) 2024-12-09T08:17:01,096 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:01,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ef14ba692d66491d959e5f9234d0e1b5_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ef14ba692d66491d959e5f9234d0e1b5_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:01,101 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/882ad01936514bffbf5a34388c9b356f, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:01,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/882ad01936514bffbf5a34388c9b356f is 175, key is test_row_0/A:col10/1733732221068/Put/seqid=0 2024-12-09T08:17:01,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742431_1607 (size=66023) 2024-12-09T08:17:01,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732221114, max timestamp=1733732221127Num uniques 859; 2024-12-09T08:17:01,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732221114, max timestamp=1733732221127Num uniques 859; 2024-12-09T08:17:01,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:17:01,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:17:01,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733732221114, max timestamp=1733732221127Num uniques 859; 2024-12-09T08:17:01,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5bda946059d1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T08:17:01,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732281159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:01,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732281160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:01,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732281271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732281271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:01,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732281480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:01,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732281480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:01,519 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/882ad01936514bffbf5a34388c9b356f 2024-12-09T08:17:01,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/33c002cf5e9d495583e434e510240356 is 50, key is test_row_0/B:col10/1733732221068/Put/seqid=0 2024-12-09T08:17:01,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742432_1608 (size=12301) 2024-12-09T08:17:01,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732281791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732281791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:01,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/33c002cf5e9d495583e434e510240356 2024-12-09T08:17:01,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/9cb85c7a35854ace983b5ea21626a2d4 is 50, key is test_row_0/C:col10/1733732221068/Put/seqid=0 2024-12-09T08:17:01,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742433_1609 (size=12301) 2024-12-09T08:17:02,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732282300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:02,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732282301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:02,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/9cb85c7a35854ace983b5ea21626a2d4 2024-12-09T08:17:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/882ad01936514bffbf5a34388c9b356f as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/882ad01936514bffbf5a34388c9b356f 2024-12-09T08:17:02,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/882ad01936514bffbf5a34388c9b356f, entries=350, sequenceid=300, filesize=64.5 K 2024-12-09T08:17:02,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/33c002cf5e9d495583e434e510240356 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/33c002cf5e9d495583e434e510240356 2024-12-09T08:17:02,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/33c002cf5e9d495583e434e510240356, entries=150, sequenceid=300, filesize=12.0 K 2024-12-09T08:17:02,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/9cb85c7a35854ace983b5ea21626a2d4 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/9cb85c7a35854ace983b5ea21626a2d4 2024-12-09T08:17:02,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/9cb85c7a35854ace983b5ea21626a2d4, entries=150, sequenceid=300, filesize=12.0 K 2024-12-09T08:17:02,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 19c9630e7ab12af49561314a634ad52e in 1283ms, sequenceid=300, compaction requested=true 2024-12-09T08:17:02,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:17:02,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T08:17:02,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:17:02,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T08:17:02,362 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:17:02,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:17:02,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19c9630e7ab12af49561314a634ad52e:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T08:17:02,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T08:17:02,362 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:17:02,363 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128963 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:17:02,363 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:17:02,363 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/A is initiating minor compaction (all files) 2024-12-09T08:17:02,363 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/B is initiating minor compaction (all files) 2024-12-09T08:17:02,363 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/A in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:02,363 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/B in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:02,363 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/55199f07c0464d6cba6fb120a087bd08, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/80a52378731b47f09c93bd2996e28f24, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/882ad01936514bffbf5a34388c9b356f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=125.9 K 2024-12-09T08:17:02,363 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/12a338d496f74c8d99e7d96fb70358da, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/1321fb4121434f5da70ff0589ad933b0, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/33c002cf5e9d495583e434e510240356] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=36.5 K 2024-12-09T08:17:02,363 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:02,363 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. files: [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/55199f07c0464d6cba6fb120a087bd08, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/80a52378731b47f09c93bd2996e28f24, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/882ad01936514bffbf5a34388c9b356f] 2024-12-09T08:17:02,363 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 12a338d496f74c8d99e7d96fb70358da, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733732218716 2024-12-09T08:17:02,363 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55199f07c0464d6cba6fb120a087bd08, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733732218716 2024-12-09T08:17:02,364 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1321fb4121434f5da70ff0589ad933b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733732218846 2024-12-09T08:17:02,364 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80a52378731b47f09c93bd2996e28f24, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733732218846 2024-12-09T08:17:02,364 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c002cf5e9d495583e434e510240356, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733732221037 2024-12-09T08:17:02,364 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] compactions.Compactor(225): Compacting 882ad01936514bffbf5a34388c9b356f, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733732221037 2024-12-09T08:17:02,367 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:02,369 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#B#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:17:02,369 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209d335aa0dee9a48059efd8ea475b29ace_19c9630e7ab12af49561314a634ad52e store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:02,369 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/40669164edab4dffb1644b4e9a94adbc is 50, key is test_row_0/B:col10/1733732221068/Put/seqid=0 2024-12-09T08:17:02,370 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209d335aa0dee9a48059efd8ea475b29ace_19c9630e7ab12af49561314a634ad52e, store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:02,371 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d335aa0dee9a48059efd8ea475b29ace_19c9630e7ab12af49561314a634ad52e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:02,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742434_1610 (size=4469) 2024-12-09T08:17:02,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742435_1611 (size=12983) 2024-12-09T08:17:02,512 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T08:17:02,775 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#A#compaction#516 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:17:02,776 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/89785846b13c4729ad1cd17c2e94a62a is 175, key is test_row_0/A:col10/1733732221068/Put/seqid=0 2024-12-09T08:17:02,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742436_1612 (size=31937) 2024-12-09T08:17:02,780 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/40669164edab4dffb1644b4e9a94adbc as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/40669164edab4dffb1644b4e9a94adbc 2024-12-09T08:17:02,784 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/B of 19c9630e7ab12af49561314a634ad52e into 40669164edab4dffb1644b4e9a94adbc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:17:02,784 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:17:02,784 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/B, priority=13, startTime=1733732222362; duration=0sec 2024-12-09T08:17:02,784 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T08:17:02,784 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:B 2024-12-09T08:17:02,784 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T08:17:02,785 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T08:17:02,785 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1541): 19c9630e7ab12af49561314a634ad52e/C is initiating minor compaction (all files) 2024-12-09T08:17:02,785 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 19c9630e7ab12af49561314a634ad52e/C in TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:02,785 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/3c08721501df4d608ecb7f3b100f134c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/1a117d1e3eb44d1892d69fddf893d435, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/9cb85c7a35854ace983b5ea21626a2d4] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp, totalSize=36.5 K 2024-12-09T08:17:02,785 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c08721501df4d608ecb7f3b100f134c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733732218716 2024-12-09T08:17:02,786 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a117d1e3eb44d1892d69fddf893d435, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733732218846 2024-12-09T08:17:02,786 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb85c7a35854ace983b5ea21626a2d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733732221037 2024-12-09T08:17:02,790 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19c9630e7ab12af49561314a634ad52e#C#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:17:02,790 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/d60505bdf64e47ac80b42fdcd7f7d504 is 50, key is test_row_0/C:col10/1733732221068/Put/seqid=0 2024-12-09T08:17:02,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742437_1613 (size=12983) 2024-12-09T08:17:02,797 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/d60505bdf64e47ac80b42fdcd7f7d504 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/d60505bdf64e47ac80b42fdcd7f7d504 2024-12-09T08:17:02,801 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/C of 19c9630e7ab12af49561314a634ad52e into d60505bdf64e47ac80b42fdcd7f7d504(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:17:02,801 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:17:02,801 INFO [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/C, priority=13, startTime=1733732222362; duration=0sec 2024-12-09T08:17:02,801 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:17:02,801 DEBUG [RS:0;5bda946059d1:43255-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:C 2024-12-09T08:17:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T08:17:02,861 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:17:02,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T08:17:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-09T08:17:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T08:17:02,863 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T08:17:02,864 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T08:17:02,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T08:17:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:02,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T08:17:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:17:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:17:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:17:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:17:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:17:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T08:17:02,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120928e8448dafa643efa695be070b7b17c4_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732222914/Put/seqid=0 2024-12-09T08:17:02,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742438_1614 (size=14994) 2024-12-09T08:17:02,924 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:02,926 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120928e8448dafa643efa695be070b7b17c4_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120928e8448dafa643efa695be070b7b17c4_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:02,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/cd7f970c017f4d18a093838c409df83c, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:02,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/cd7f970c017f4d18a093838c409df83c is 175, key is test_row_0/A:col10/1733732222914/Put/seqid=0 2024-12-09T08:17:02,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742439_1615 (size=39949) 2024-12-09T08:17:02,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732282933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:02,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:02,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:02,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:02,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732282935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:02,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49686 deadline: 1733732282941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:02,945 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_2' in 19c9630e7ab12af49561314a634ad52e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:17:02,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:02,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:02,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T08:17:03,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T08:17:03,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:17:03,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732283041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:03,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732283051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:03,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T08:17:03,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:17:03,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T08:17:03,183 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/89785846b13c4729ad1cd17c2e94a62a as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/89785846b13c4729ad1cd17c2e94a62a 2024-12-09T08:17:03,186 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 19c9630e7ab12af49561314a634ad52e/A of 19c9630e7ab12af49561314a634ad52e into 89785846b13c4729ad1cd17c2e94a62a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:17:03,186 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:17:03,186 INFO [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., storeName=19c9630e7ab12af49561314a634ad52e/A, priority=13, startTime=1733732222362; duration=0sec 2024-12-09T08:17:03,186 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T08:17:03,186 DEBUG [RS:0;5bda946059d1:43255-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19c9630e7ab12af49561314a634ad52e:A 2024-12-09T08:17:03,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732283250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:03,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732283261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:03,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1733732283311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:03,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49644 deadline: 1733732283311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:03,314 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_1' in 19c9630e7ab12af49561314a634ad52e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:17:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,314 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5bda946059d1,43255,1733732044053 for 'test_row_0' in 19c9630e7ab12af49561314a634ad52e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-12-09T08:17:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:03,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T08:17:03,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:17:03,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,330 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/cd7f970c017f4d18a093838c409df83c 2024-12-09T08:17:03,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/5f7281b0889840ac9f47ecd70c6ca4db is 50, key is test_row_0/B:col10/1733732222914/Put/seqid=0 2024-12-09T08:17:03,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742440_1616 (size=12301) 2024-12-09T08:17:03,449 INFO [Thread-2374 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2374. 2024-12-09T08:17:03,449 DEBUG [Thread-2374 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:17:03,449 DEBUG [Thread-2374 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:03,449 DEBUG [Thread-2374 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:03,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:17:03,450 INFO [Thread-2378 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2378. 2024-12-09T08:17:03,450 DEBUG [Thread-2378 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:17:03,450 DEBUG [Thread-2378 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:03,450 DEBUG [Thread-2378 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:03,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:17:03,451 INFO [Thread-2382 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2382. 2024-12-09T08:17:03,451 DEBUG [Thread-2382 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:17:03,451 DEBUG [Thread-2382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:03,451 DEBUG [Thread-2382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:03,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:17:03,453 INFO [Thread-2380 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2380. 2024-12-09T08:17:03,453 DEBUG [Thread-2380 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:17:03,453 DEBUG [Thread-2380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:03,453 DEBUG [Thread-2380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:03,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:17:03,454 INFO [Thread-2376 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2376. 2024-12-09T08:17:03,454 DEBUG [Thread-2376 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:17:03,454 DEBUG [Thread-2376 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:03,454 DEBUG [Thread-2376 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:03,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:17:03,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T08:17:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:17:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T08:17:03,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732283561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:03,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732283571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:03,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T08:17:03,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:17:03,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/5f7281b0889840ac9f47ecd70c6ca4db 2024-12-09T08:17:03,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/65d016ebc5394d4eb33ba1abd4659c02 is 50, key is test_row_0/C:col10/1733732222914/Put/seqid=0 2024-12-09T08:17:03,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742441_1617 (size=12301) 2024-12-09T08:17:03,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T08:17:03,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:17:03,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T08:17:03,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:17:03,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:03,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:04,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T08:17:04,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49716 deadline: 1733732284071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:04,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:04,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:04,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:04,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T08:17:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49662 deadline: 1733732284081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 2024-12-09T08:17:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 , the old value is region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T08:17:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., hostname=5bda946059d1,43255,1733732044053, seqNum=5 because the exception is null or not the one we care about 2024-12-09T08:17:04,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T08:17:04,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:04,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:17:04,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:04,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:04,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:04,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/65d016ebc5394d4eb33ba1abd4659c02 2024-12-09T08:17:04,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/cd7f970c017f4d18a093838c409df83c as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/cd7f970c017f4d18a093838c409df83c 2024-12-09T08:17:04,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/cd7f970c017f4d18a093838c409df83c, entries=200, sequenceid=338, filesize=39.0 K 2024-12-09T08:17:04,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/5f7281b0889840ac9f47ecd70c6ca4db as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/5f7281b0889840ac9f47ecd70c6ca4db 2024-12-09T08:17:04,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/5f7281b0889840ac9f47ecd70c6ca4db, entries=150, sequenceid=338, filesize=12.0 K 2024-12-09T08:17:04,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/65d016ebc5394d4eb33ba1abd4659c02 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/65d016ebc5394d4eb33ba1abd4659c02 2024-12-09T08:17:04,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/65d016ebc5394d4eb33ba1abd4659c02, entries=150, sequenceid=338, filesize=12.0 K 2024-12-09T08:17:04,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 19c9630e7ab12af49561314a634ad52e in 1255ms, sequenceid=338, compaction requested=false 2024-12-09T08:17:04,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:17:04,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43255 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T08:17:04,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:04,238 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T08:17:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:17:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:17:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:17:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:17:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:17:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:17:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092f883dc718ec40ada5fbe065beb128f2_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732222934/Put/seqid=0 2024-12-09T08:17:04,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742442_1618 (size=9914) 2024-12-09T08:17:04,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:04,650 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092f883dc718ec40ada5fbe065beb128f2_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092f883dc718ec40ada5fbe065beb128f2_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:04,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/ed1223615a1f4e8f95d1ef6a38056901, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:04,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/ed1223615a1f4e8f95d1ef6a38056901 is 175, key is test_row_0/A:col10/1733732222934/Put/seqid=0 2024-12-09T08:17:04,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742443_1619 (size=22561) 2024-12-09T08:17:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T08:17:05,055 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/ed1223615a1f4e8f95d1ef6a38056901 2024-12-09T08:17:05,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/e2b088eaa8e349fc89242f15aab8c7e1 is 50, key is test_row_0/B:col10/1733732222934/Put/seqid=0 2024-12-09T08:17:05,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742444_1620 (size=9857) 2024-12-09T08:17:05,067 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/e2b088eaa8e349fc89242f15aab8c7e1 2024-12-09T08:17:05,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/116468f1e0f54db6bbad499bd8a25e8d is 50, key is test_row_0/C:col10/1733732222934/Put/seqid=0 2024-12-09T08:17:05,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742445_1621 (size=9857) 2024-12-09T08:17:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43255 {}] regionserver.HRegion(8855): Flush requested on 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:05,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. as already flushing 2024-12-09T08:17:05,082 INFO [Thread-2367 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2367. 2024-12-09T08:17:05,082 DEBUG [Thread-2367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:17:05,082 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:05,082 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:05,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:17:05,092 INFO [Thread-2371 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2371. 2024-12-09T08:17:05,092 DEBUG [Thread-2371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:17:05,092 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:05,092 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:05,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:17:05,324 INFO [Thread-2365 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2365. 2024-12-09T08:17:05,324 DEBUG [Thread-2365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:17:05,324 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:05,324 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:05,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:17:05,332 INFO [Thread-2369 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2369. 2024-12-09T08:17:05,332 DEBUG [Thread-2369 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:17:05,332 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:05,332 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:05,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:17:05,479 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/116468f1e0f54db6bbad499bd8a25e8d 2024-12-09T08:17:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/ed1223615a1f4e8f95d1ef6a38056901 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/ed1223615a1f4e8f95d1ef6a38056901 2024-12-09T08:17:05,485 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/ed1223615a1f4e8f95d1ef6a38056901, entries=100, sequenceid=349, filesize=22.0 K 2024-12-09T08:17:05,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/e2b088eaa8e349fc89242f15aab8c7e1 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/e2b088eaa8e349fc89242f15aab8c7e1 2024-12-09T08:17:05,489 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/e2b088eaa8e349fc89242f15aab8c7e1, entries=100, sequenceid=349, filesize=9.6 K 2024-12-09T08:17:05,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/116468f1e0f54db6bbad499bd8a25e8d as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/116468f1e0f54db6bbad499bd8a25e8d 2024-12-09T08:17:05,493 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/116468f1e0f54db6bbad499bd8a25e8d, entries=100, sequenceid=349, filesize=9.6 K 2024-12-09T08:17:05,494 INFO [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=26.84 KB/27480 for 19c9630e7ab12af49561314a634ad52e in 1256ms, sequenceid=349, compaction requested=true 2024-12-09T08:17:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 19c9630e7ab12af49561314a634ad52e: 2024-12-09T08:17:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-09T08:17:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-09T08:17:05,496 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-09T08:17:05,496 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6310 sec 2024-12-09T08:17:05,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.6350 sec 2024-12-09T08:17:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T08:17:07,021 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:17:13,056 INFO [Thread-2363 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2363. 2024-12-09T08:17:13,056 DEBUG [Thread-2363 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T08:17:13,056 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:13,057 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:13,057 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:17:13,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T08:17:13,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-12-09T08:17:13,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-09T08:17:13,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-09T08:17:13,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-12-09T08:17:13,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-12-09T08:17:13,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T08:17:13,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4329 2024-12-09T08:17:13,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4141 2024-12-09T08:17:13,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4138 2024-12-09T08:17:13,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4357 2024-12-09T08:17:13,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4128 2024-12-09T08:17:13,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T08:17:13,059 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T08:17:13,059 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:17:13,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:13,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:13,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:17:13,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T08:17:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T08:17:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T08:17:13,066 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732233066"}]},"ts":"1733732233066"} 2024-12-09T08:17:13,067 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T08:17:13,067 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T08:17:13,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T08:17:13,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, UNASSIGN}] 2024-12-09T08:17:13,070 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, UNASSIGN 2024-12-09T08:17:13,071 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=19c9630e7ab12af49561314a634ad52e, regionState=CLOSING, regionLocation=5bda946059d1,43255,1733732044053 2024-12-09T08:17:13,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, UNASSIGN because future has completed 2024-12-09T08:17:13,072 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:17:13,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053}] 2024-12-09T08:17:13,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T08:17:13,224 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:13,225 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:17:13,225 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 19c9630e7ab12af49561314a634ad52e, disabling compactions & flushes 2024-12-09T08:17:13,225 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:13,225 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:13,225 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. after waiting 0 ms 2024-12-09T08:17:13,225 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:13,225 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 19c9630e7ab12af49561314a634ad52e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T08:17:13,225 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=A 2024-12-09T08:17:13,225 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:17:13,225 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=B 2024-12-09T08:17:13,226 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:17:13,226 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19c9630e7ab12af49561314a634ad52e, store=C 2024-12-09T08:17:13,226 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T08:17:13,234 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098d722cd45f474a5a914146a1bd254503_19c9630e7ab12af49561314a634ad52e is 50, key is test_row_0/A:col10/1733732225322/Put/seqid=0 2024-12-09T08:17:13,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742446_1622 (size=12454) 2024-12-09T08:17:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T08:17:13,639 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:13,650 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098d722cd45f474a5a914146a1bd254503_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098d722cd45f474a5a914146a1bd254503_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:13,652 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/cb9c299ce0f644318d56927dbc1f58e8, store: [table=TestAcidGuarantees family=A region=19c9630e7ab12af49561314a634ad52e] 2024-12-09T08:17:13,652 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/cb9c299ce0f644318d56927dbc1f58e8 is 175, key is test_row_0/A:col10/1733732225322/Put/seqid=0 2024-12-09T08:17:13,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742447_1623 (size=31255) 2024-12-09T08:17:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T08:17:14,057 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/cb9c299ce0f644318d56927dbc1f58e8 2024-12-09T08:17:14,069 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/b640a4dccb43411ab8d04a6f28b03bfa is 50, key is test_row_0/B:col10/1733732225322/Put/seqid=0 2024-12-09T08:17:14,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742448_1624 (size=12301) 2024-12-09T08:17:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T08:17:14,472 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/b640a4dccb43411ab8d04a6f28b03bfa 2024-12-09T08:17:14,479 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/c5085a3c12714821bd537913325b76e2 is 50, key is test_row_0/C:col10/1733732225322/Put/seqid=0 2024-12-09T08:17:14,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742449_1625 (size=12301) 2024-12-09T08:17:14,883 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/c5085a3c12714821bd537913325b76e2 2024-12-09T08:17:14,892 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/A/cb9c299ce0f644318d56927dbc1f58e8 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/cb9c299ce0f644318d56927dbc1f58e8 2024-12-09T08:17:14,896 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/cb9c299ce0f644318d56927dbc1f58e8, entries=150, sequenceid=357, filesize=30.5 K 2024-12-09T08:17:14,897 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/B/b640a4dccb43411ab8d04a6f28b03bfa as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/b640a4dccb43411ab8d04a6f28b03bfa 2024-12-09T08:17:14,902 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/b640a4dccb43411ab8d04a6f28b03bfa, entries=150, sequenceid=357, filesize=12.0 K 2024-12-09T08:17:14,903 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/.tmp/C/c5085a3c12714821bd537913325b76e2 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/c5085a3c12714821bd537913325b76e2 2024-12-09T08:17:14,907 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/c5085a3c12714821bd537913325b76e2, entries=150, sequenceid=357, filesize=12.0 K 2024-12-09T08:17:14,907 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 19c9630e7ab12af49561314a634ad52e in 1682ms, sequenceid=357, compaction requested=true 2024-12-09T08:17:14,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/59e425a888b64f518bebe8d783f17346, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/bef94e8ba03945e08c48e5ba6ae40800, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/d31915d9ff96406bb32a259408b35c0c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/7d29fd9976444a64b76d09704a2ad233, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/9e443f9226004a17baea8c6d91e46329, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/629974eb0f234d839358d799675b874f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/2ad158fac58e435e8f28a90c4df541c5, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/c7b45edc7e2c4ddb9d80e416f600f0b9, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/03bf27b2815b4621bad4d92938a4915a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/0cc1df8ebb7d4bfc8785bf85dbef9fa7, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/b1d0158d520243c0adbcf65af04844dd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/25df820ae3f9433085ace54f243b3c6d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/857df265ffde4cd19b8eae474b606498, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/34e576901d3b4ba8a9c7a08e45fefa2b, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/fc05f73ceeb8492e8e57fa72d6cbc59c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/ad7b94129d594d879c8f2b95d54b2a7c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/6e69ab2956674208928a846c8a4d85d4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/55199f07c0464d6cba6fb120a087bd08, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/80a52378731b47f09c93bd2996e28f24, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/882ad01936514bffbf5a34388c9b356f] to archive 2024-12-09T08:17:14,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:17:14,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/59e425a888b64f518bebe8d783f17346 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/59e425a888b64f518bebe8d783f17346 2024-12-09T08:17:14,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/bef94e8ba03945e08c48e5ba6ae40800 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/bef94e8ba03945e08c48e5ba6ae40800 2024-12-09T08:17:14,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/d31915d9ff96406bb32a259408b35c0c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/d31915d9ff96406bb32a259408b35c0c 2024-12-09T08:17:14,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/7d29fd9976444a64b76d09704a2ad233 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/7d29fd9976444a64b76d09704a2ad233 2024-12-09T08:17:14,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/9e443f9226004a17baea8c6d91e46329 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/9e443f9226004a17baea8c6d91e46329 2024-12-09T08:17:14,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/629974eb0f234d839358d799675b874f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/629974eb0f234d839358d799675b874f 2024-12-09T08:17:14,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/2ad158fac58e435e8f28a90c4df541c5 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/2ad158fac58e435e8f28a90c4df541c5 2024-12-09T08:17:14,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/c7b45edc7e2c4ddb9d80e416f600f0b9 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/c7b45edc7e2c4ddb9d80e416f600f0b9 2024-12-09T08:17:14,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/03bf27b2815b4621bad4d92938a4915a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/03bf27b2815b4621bad4d92938a4915a 2024-12-09T08:17:14,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/0cc1df8ebb7d4bfc8785bf85dbef9fa7 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/0cc1df8ebb7d4bfc8785bf85dbef9fa7 2024-12-09T08:17:14,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/b1d0158d520243c0adbcf65af04844dd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/b1d0158d520243c0adbcf65af04844dd 2024-12-09T08:17:14,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/25df820ae3f9433085ace54f243b3c6d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/25df820ae3f9433085ace54f243b3c6d 2024-12-09T08:17:14,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/857df265ffde4cd19b8eae474b606498 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/857df265ffde4cd19b8eae474b606498 2024-12-09T08:17:14,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/34e576901d3b4ba8a9c7a08e45fefa2b to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/34e576901d3b4ba8a9c7a08e45fefa2b 2024-12-09T08:17:14,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/fc05f73ceeb8492e8e57fa72d6cbc59c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/fc05f73ceeb8492e8e57fa72d6cbc59c 2024-12-09T08:17:14,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/ad7b94129d594d879c8f2b95d54b2a7c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/ad7b94129d594d879c8f2b95d54b2a7c 2024-12-09T08:17:14,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/6e69ab2956674208928a846c8a4d85d4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/6e69ab2956674208928a846c8a4d85d4 2024-12-09T08:17:14,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/55199f07c0464d6cba6fb120a087bd08 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/55199f07c0464d6cba6fb120a087bd08 2024-12-09T08:17:14,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/80a52378731b47f09c93bd2996e28f24 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/80a52378731b47f09c93bd2996e28f24 2024-12-09T08:17:14,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/882ad01936514bffbf5a34388c9b356f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/882ad01936514bffbf5a34388c9b356f 2024-12-09T08:17:14,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/7172f2786db343d3b3ac8c7f9853a652, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/bf4903566eb446edbfc51cac3893221e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/5cde1e7a8b7f430e93a45872c151b69a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/2a3921f549574652b8976b83025875a3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/8f7d5c853c7e488ca77b48ae01b067e4, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/74a659e4520d4e1596aa019edf38b18c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/55fc59b65f3e4ca0bb1df87a621cba6a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/49c02ae75e314c47a7e9e4ffb5ec9485, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/217154932ee84bd999926c5c67b1b694, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/04d5f844a69e444baf48ffda7fd192f8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/0b4fac49e73c4fe9b3fe8dca8b3abf03, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/ebfbe3dadb5a433294405c4e33681875, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/6f7b680684b544bb84a26b71b94df2bf, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/ceff3cf38bf84fa2b6acab612bd6f925, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/e0176dce0a414276acac1f93e6e12304, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/718d39c876184204ae3d5fb8d98e8abf, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/12a338d496f74c8d99e7d96fb70358da, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/01e9b972dfcf4c5892c58268c91ea454, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/1321fb4121434f5da70ff0589ad933b0, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/33c002cf5e9d495583e434e510240356] to archive 2024-12-09T08:17:14,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:17:14,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/7172f2786db343d3b3ac8c7f9853a652 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/7172f2786db343d3b3ac8c7f9853a652 2024-12-09T08:17:14,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/bf4903566eb446edbfc51cac3893221e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/bf4903566eb446edbfc51cac3893221e 2024-12-09T08:17:14,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/5cde1e7a8b7f430e93a45872c151b69a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/5cde1e7a8b7f430e93a45872c151b69a 2024-12-09T08:17:14,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/2a3921f549574652b8976b83025875a3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/2a3921f549574652b8976b83025875a3 2024-12-09T08:17:14,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/8f7d5c853c7e488ca77b48ae01b067e4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/8f7d5c853c7e488ca77b48ae01b067e4 2024-12-09T08:17:14,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/74a659e4520d4e1596aa019edf38b18c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/74a659e4520d4e1596aa019edf38b18c 2024-12-09T08:17:14,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/55fc59b65f3e4ca0bb1df87a621cba6a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/55fc59b65f3e4ca0bb1df87a621cba6a 2024-12-09T08:17:14,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/49c02ae75e314c47a7e9e4ffb5ec9485 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/49c02ae75e314c47a7e9e4ffb5ec9485 2024-12-09T08:17:14,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/217154932ee84bd999926c5c67b1b694 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/217154932ee84bd999926c5c67b1b694 2024-12-09T08:17:14,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/04d5f844a69e444baf48ffda7fd192f8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/04d5f844a69e444baf48ffda7fd192f8 2024-12-09T08:17:14,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/0b4fac49e73c4fe9b3fe8dca8b3abf03 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/0b4fac49e73c4fe9b3fe8dca8b3abf03 2024-12-09T08:17:14,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/ebfbe3dadb5a433294405c4e33681875 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/ebfbe3dadb5a433294405c4e33681875 2024-12-09T08:17:14,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/6f7b680684b544bb84a26b71b94df2bf to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/6f7b680684b544bb84a26b71b94df2bf 2024-12-09T08:17:14,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/ceff3cf38bf84fa2b6acab612bd6f925 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/ceff3cf38bf84fa2b6acab612bd6f925 2024-12-09T08:17:14,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/e0176dce0a414276acac1f93e6e12304 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/e0176dce0a414276acac1f93e6e12304 2024-12-09T08:17:14,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/718d39c876184204ae3d5fb8d98e8abf to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/718d39c876184204ae3d5fb8d98e8abf 2024-12-09T08:17:14,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/12a338d496f74c8d99e7d96fb70358da to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/12a338d496f74c8d99e7d96fb70358da 2024-12-09T08:17:14,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/01e9b972dfcf4c5892c58268c91ea454 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/01e9b972dfcf4c5892c58268c91ea454 2024-12-09T08:17:14,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/1321fb4121434f5da70ff0589ad933b0 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/1321fb4121434f5da70ff0589ad933b0 2024-12-09T08:17:14,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/33c002cf5e9d495583e434e510240356 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/33c002cf5e9d495583e434e510240356 2024-12-09T08:17:14,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/857ed0af0f894e11a2629b62bd06942d, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/8f174d9ba24b4ca0b5e1505f2fcbd605, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/645c063d0f9f4807aa144cb90361ab2f, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/f3f4ea13ae0f44e597c67133bce8533e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/8f004061cba5402a8e5eee57bc7126b3, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/db64cac4a61e48ccaea3ad951832c048, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/b9811d4fed08470fb678379f7bc9c4bb, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/30254500f5674510835ba7c43ae62866, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ee3dc16c573349ecae707b5436f3f2a8, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/2365d2f22cfa4102b987ac3de0807671, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/a3f5624ac0eb41b48a7e638172a316ab, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/5793c0ce42474efd9756a87ab6f32dfd, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/829e9ff40e14424cbb9c1f818bc6d83a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/cc94b8e67937485b98062a26b4add137, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ef0d9294386d45a8b32eeb68712aa724, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ac91a13972fb4673a5d61fe1acc2b68a, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/3c08721501df4d608ecb7f3b100f134c, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/3504b125f26f45799d068ef78926b33e, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/1a117d1e3eb44d1892d69fddf893d435, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/9cb85c7a35854ace983b5ea21626a2d4] to archive 2024-12-09T08:17:14,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:17:14,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/857ed0af0f894e11a2629b62bd06942d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/857ed0af0f894e11a2629b62bd06942d 2024-12-09T08:17:14,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/8f174d9ba24b4ca0b5e1505f2fcbd605 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/8f174d9ba24b4ca0b5e1505f2fcbd605 2024-12-09T08:17:14,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/645c063d0f9f4807aa144cb90361ab2f to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/645c063d0f9f4807aa144cb90361ab2f 2024-12-09T08:17:14,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/f3f4ea13ae0f44e597c67133bce8533e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/f3f4ea13ae0f44e597c67133bce8533e 2024-12-09T08:17:14,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/8f004061cba5402a8e5eee57bc7126b3 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/8f004061cba5402a8e5eee57bc7126b3 2024-12-09T08:17:14,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/db64cac4a61e48ccaea3ad951832c048 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/db64cac4a61e48ccaea3ad951832c048 2024-12-09T08:17:14,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/b9811d4fed08470fb678379f7bc9c4bb to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/b9811d4fed08470fb678379f7bc9c4bb 2024-12-09T08:17:14,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/30254500f5674510835ba7c43ae62866 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/30254500f5674510835ba7c43ae62866 2024-12-09T08:17:14,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ee3dc16c573349ecae707b5436f3f2a8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ee3dc16c573349ecae707b5436f3f2a8 2024-12-09T08:17:14,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/2365d2f22cfa4102b987ac3de0807671 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/2365d2f22cfa4102b987ac3de0807671 2024-12-09T08:17:14,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/a3f5624ac0eb41b48a7e638172a316ab to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/a3f5624ac0eb41b48a7e638172a316ab 2024-12-09T08:17:14,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/5793c0ce42474efd9756a87ab6f32dfd to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/5793c0ce42474efd9756a87ab6f32dfd 2024-12-09T08:17:14,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/829e9ff40e14424cbb9c1f818bc6d83a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/829e9ff40e14424cbb9c1f818bc6d83a 2024-12-09T08:17:14,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/cc94b8e67937485b98062a26b4add137 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/cc94b8e67937485b98062a26b4add137 2024-12-09T08:17:14,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ef0d9294386d45a8b32eeb68712aa724 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ef0d9294386d45a8b32eeb68712aa724 2024-12-09T08:17:14,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ac91a13972fb4673a5d61fe1acc2b68a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/ac91a13972fb4673a5d61fe1acc2b68a 2024-12-09T08:17:14,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/3c08721501df4d608ecb7f3b100f134c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/3c08721501df4d608ecb7f3b100f134c 2024-12-09T08:17:14,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/3504b125f26f45799d068ef78926b33e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/3504b125f26f45799d068ef78926b33e 2024-12-09T08:17:14,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/1a117d1e3eb44d1892d69fddf893d435 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/1a117d1e3eb44d1892d69fddf893d435 2024-12-09T08:17:14,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/9cb85c7a35854ace983b5ea21626a2d4 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/9cb85c7a35854ace983b5ea21626a2d4 2024-12-09T08:17:14,962 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/recovered.edits/360.seqid, newMaxSeqId=360, maxSeqId=4 2024-12-09T08:17:14,963 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. 2024-12-09T08:17:14,963 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 19c9630e7ab12af49561314a634ad52e: Waiting for close lock at 1733732233225Running coprocessor pre-close hooks at 1733732233225Disabling compacts and flushes for region at 1733732233225Disabling writes for close at 1733732233225Obtaining lock to block concurrent updates at 1733732233225Preparing flush snapshotting stores in 19c9630e7ab12af49561314a634ad52e at 1733732233225Finished memstore snapshotting TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733732233226 (+1 ms)Flushing stores of TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e. at 1733732233226Flushing 19c9630e7ab12af49561314a634ad52e/A: creating writer at 1733732233226Flushing mob file 19c9630e7ab12af49561314a634ad52e/A: appending metadata at 1733732233234 (+8 ms)Flushing mob file 19c9630e7ab12af49561314a634ad52e/A: closing flushed file at 1733732233234Flushing 19c9630e7ab12af49561314a634ad52e/A: appending metadata at 1733732233652 (+418 ms)Flushing 19c9630e7ab12af49561314a634ad52e/A: closing flushed file at 1733732233652Flushing 19c9630e7ab12af49561314a634ad52e/B: creating writer at 1733732234064 (+412 ms)Flushing 19c9630e7ab12af49561314a634ad52e/B: appending metadata at 1733732234069 (+5 ms)Flushing 19c9630e7ab12af49561314a634ad52e/B: closing flushed file at 1733732234069Flushing 19c9630e7ab12af49561314a634ad52e/C: creating writer at 1733732234475 (+406 ms)Flushing 19c9630e7ab12af49561314a634ad52e/C: appending metadata at 1733732234479 (+4 ms)Flushing 19c9630e7ab12af49561314a634ad52e/C: closing flushed file at 1733732234479Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@344dd4c0: reopening flushed file at 1733732234890 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25968c9d: reopening flushed file at 1733732234897 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37290a3: reopening flushed file at 1733732234902 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 19c9630e7ab12af49561314a634ad52e in 1682ms, sequenceid=357, compaction requested=true at 1733732234908 (+6 ms)Writing region close event to WAL at 1733732234960 (+52 ms)Running coprocessor post-close hooks at 1733732234963 (+3 ms)Closed at 1733732234963 2024-12-09T08:17:14,964 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:14,965 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=19c9630e7ab12af49561314a634ad52e, regionState=CLOSED 2024-12-09T08:17:14,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 because future has completed 2024-12-09T08:17:14,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-09T08:17:14,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 19c9630e7ab12af49561314a634ad52e, server=5bda946059d1,43255,1733732044053 in 1.8940 sec 2024-12-09T08:17:14,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-12-09T08:17:14,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=19c9630e7ab12af49561314a634ad52e, UNASSIGN in 1.8980 sec 2024-12-09T08:17:14,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-09T08:17:14,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9010 sec 2024-12-09T08:17:14,971 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732234971"}]},"ts":"1733732234971"} 2024-12-09T08:17:14,972 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T08:17:14,972 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T08:17:14,973 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9100 sec 2024-12-09T08:17:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T08:17:15,211 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:17:15,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T08:17:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:17:15,213 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:17:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T08:17:15,214 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:17:15,216 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,218 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/recovered.edits] 2024-12-09T08:17:15,220 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/89785846b13c4729ad1cd17c2e94a62a to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/89785846b13c4729ad1cd17c2e94a62a 2024-12-09T08:17:15,221 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/cb9c299ce0f644318d56927dbc1f58e8 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/cb9c299ce0f644318d56927dbc1f58e8 2024-12-09T08:17:15,222 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/cd7f970c017f4d18a093838c409df83c to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/cd7f970c017f4d18a093838c409df83c 2024-12-09T08:17:15,223 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/ed1223615a1f4e8f95d1ef6a38056901 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/A/ed1223615a1f4e8f95d1ef6a38056901 2024-12-09T08:17:15,225 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/40669164edab4dffb1644b4e9a94adbc to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/40669164edab4dffb1644b4e9a94adbc 2024-12-09T08:17:15,226 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/5f7281b0889840ac9f47ecd70c6ca4db to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/5f7281b0889840ac9f47ecd70c6ca4db 2024-12-09T08:17:15,227 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/b640a4dccb43411ab8d04a6f28b03bfa to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/b640a4dccb43411ab8d04a6f28b03bfa 2024-12-09T08:17:15,227 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/e2b088eaa8e349fc89242f15aab8c7e1 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/B/e2b088eaa8e349fc89242f15aab8c7e1 2024-12-09T08:17:15,229 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/116468f1e0f54db6bbad499bd8a25e8d to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/116468f1e0f54db6bbad499bd8a25e8d 2024-12-09T08:17:15,229 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/65d016ebc5394d4eb33ba1abd4659c02 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/65d016ebc5394d4eb33ba1abd4659c02 2024-12-09T08:17:15,230 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/c5085a3c12714821bd537913325b76e2 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/c5085a3c12714821bd537913325b76e2 2024-12-09T08:17:15,230 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/d60505bdf64e47ac80b42fdcd7f7d504 to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/C/d60505bdf64e47ac80b42fdcd7f7d504 2024-12-09T08:17:15,232 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/recovered.edits/360.seqid to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e/recovered.edits/360.seqid 2024-12-09T08:17:15,232 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/default/TestAcidGuarantees/19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,232 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T08:17:15,233 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T08:17:15,233 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T08:17:15,235 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091777a717815546db95c32600156be55c_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091777a717815546db95c32600156be55c_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,236 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120927d0368c4a2c4b85baf9eb667964839b_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120927d0368c4a2c4b85baf9eb667964839b_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,236 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120928e8448dafa643efa695be070b7b17c4_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120928e8448dafa643efa695be070b7b17c4_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,237 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092f883dc718ec40ada5fbe065beb128f2_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092f883dc718ec40ada5fbe065beb128f2_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,238 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120943994c13b1fb42a0a2d6769fc177ffd5_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120943994c13b1fb42a0a2d6769fc177ffd5_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,238 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095f3d9a11e12c4f6dbafdcd09240f2af1_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095f3d9a11e12c4f6dbafdcd09240f2af1_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,239 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096526fde6838344ea856be771631555df_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096526fde6838344ea856be771631555df_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,239 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098392c76ea0a04694a486a8f51360a5d1_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098392c76ea0a04694a486a8f51360a5d1_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,240 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098b5beed2efa241b3b8fa5873766a79e9_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098b5beed2efa241b3b8fa5873766a79e9_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,241 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098d722cd45f474a5a914146a1bd254503_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098d722cd45f474a5a914146a1bd254503_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,241 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099eac5a3fb21d472697053dd0f296f18c_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099eac5a3fb21d472697053dd0f296f18c_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,242 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a53d1d9a6b054ef98f3414823088e258_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a53d1d9a6b054ef98f3414823088e258_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,243 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d35dad6468b94776949a77bbc4844d75_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d35dad6468b94776949a77bbc4844d75_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,243 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d5e6715da1264af2a2a52e952f724757_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d5e6715da1264af2a2a52e952f724757_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,244 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e6198f1967d345f99df1ee01c9ce91b5_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e6198f1967d345f99df1ee01c9ce91b5_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,244 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ea724410bf4e49db9366feee0ee264a3_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ea724410bf4e49db9366feee0ee264a3_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,245 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ea955e168aed4dcca6e7a9b1aa905ee1_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ea955e168aed4dcca6e7a9b1aa905ee1_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,246 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ef14ba692d66491d959e5f9234d0e1b5_19c9630e7ab12af49561314a634ad52e to hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ef14ba692d66491d959e5f9234d0e1b5_19c9630e7ab12af49561314a634ad52e 2024-12-09T08:17:15,246 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T08:17:15,247 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:17:15,249 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T08:17:15,251 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T08:17:15,252 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:17:15,252 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T08:17:15,252 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733732235252"}]},"ts":"9223372036854775807"} 2024-12-09T08:17:15,253 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T08:17:15,253 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 19c9630e7ab12af49561314a634ad52e, NAME => 'TestAcidGuarantees,,1733732201304.19c9630e7ab12af49561314a634ad52e.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T08:17:15,253 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T08:17:15,253 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733732235253"}]},"ts":"9223372036854775807"} 2024-12-09T08:17:15,255 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T08:17:15,255 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T08:17:15,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-12-09T08:17:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T08:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T08:17:15,322 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T08:17:15,336 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 249), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=451 (was 421) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4101 (was 4443) 2024-12-09T08:17:15,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T08:17:15,337 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T08:17:15,337 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:17:15,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:15,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:15,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:17:15,337 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T08:17:15,337 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1945866401, stopped=false 2024-12-09T08:17:15,338 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5bda946059d1,45611,1733732043366 2024-12-09T08:17:15,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T08:17:15,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T08:17:15,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:17:15,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:17:15,376 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T08:17:15,377 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T08:17:15,377 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:17:15,378 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:17:15,378 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:17:15,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:15,379 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5bda946059d1,43255,1733732044053' ***** 2024-12-09T08:17:15,379 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T08:17:15,380 INFO [RS:0;5bda946059d1:43255 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T08:17:15,380 INFO [RS:0;5bda946059d1:43255 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T08:17:15,380 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T08:17:15,380 INFO [RS:0;5bda946059d1:43255 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T08:17:15,380 INFO [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(959): stopping server 5bda946059d1,43255,1733732044053 2024-12-09T08:17:15,380 INFO [RS:0;5bda946059d1:43255 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T08:17:15,381 INFO [RS:0;5bda946059d1:43255 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5bda946059d1:43255. 2024-12-09T08:17:15,381 DEBUG [RS:0;5bda946059d1:43255 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:17:15,381 DEBUG [RS:0;5bda946059d1:43255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:15,381 INFO [RS:0;5bda946059d1:43255 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T08:17:15,381 INFO [RS:0;5bda946059d1:43255 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T08:17:15,381 INFO [RS:0;5bda946059d1:43255 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T08:17:15,381 INFO [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T08:17:15,383 INFO [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T08:17:15,383 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T08:17:15,383 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T08:17:15,383 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T08:17:15,383 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T08:17:15,383 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T08:17:15,383 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T08:17:15,383 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T08:17:15,383 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-09T08:17:15,404 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/.tmp/info/147eef839ba5484aa62b31887f42d659 is 95, key is TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1./info:/1733732081070/DeleteFamily/seqid=0 2024-12-09T08:17:15,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742450_1626 (size=6514) 2024-12-09T08:17:15,446 INFO [regionserver/5bda946059d1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:17:15,583 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T08:17:15,783 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T08:17:15,808 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/.tmp/info/147eef839ba5484aa62b31887f42d659 2024-12-09T08:17:15,833 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/.tmp/ns/e3790449c3ae4ee8aa57de527707a6cd is 93, key is TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1./ns:/1733732081033/DeleteFamily/seqid=0 2024-12-09T08:17:15,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742451_1627 (size=5906) 2024-12-09T08:17:15,984 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T08:17:16,032 INFO [regionserver/5bda946059d1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T08:17:16,032 INFO [regionserver/5bda946059d1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T08:17:16,184 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T08:17:16,236 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/.tmp/ns/e3790449c3ae4ee8aa57de527707a6cd 2024-12-09T08:17:16,251 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/.tmp/rep_barrier/a6d5969b3eaf4d8baef6bfd7a5bed22e is 102, key is TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1./rep_barrier:/1733732081033/DeleteFamily/seqid=0 2024-12-09T08:17:16,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742452_1628 (size=6025) 2024-12-09T08:17:16,384 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T08:17:16,585 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T08:17:16,655 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/.tmp/rep_barrier/a6d5969b3eaf4d8baef6bfd7a5bed22e 2024-12-09T08:17:16,677 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/.tmp/table/c5bd80aea8d347a0a5522ecaddf729bb is 96, key is TestAcidGuarantees,,1733732048848.589b933f5d38176578d90952a67fb0a1./table:/1733732081033/DeleteFamily/seqid=0 2024-12-09T08:17:16,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742453_1629 (size=5936) 2024-12-09T08:17:16,785 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T08:17:16,985 DEBUG [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T08:17:17,081 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/.tmp/table/c5bd80aea8d347a0a5522ecaddf729bb 2024-12-09T08:17:17,084 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/.tmp/info/147eef839ba5484aa62b31887f42d659 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/info/147eef839ba5484aa62b31887f42d659 2024-12-09T08:17:17,087 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/info/147eef839ba5484aa62b31887f42d659, entries=12, sequenceid=90, filesize=6.4 K 2024-12-09T08:17:17,088 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/.tmp/ns/e3790449c3ae4ee8aa57de527707a6cd as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/ns/e3790449c3ae4ee8aa57de527707a6cd 2024-12-09T08:17:17,090 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/ns/e3790449c3ae4ee8aa57de527707a6cd, entries=8, sequenceid=90, filesize=5.8 K 2024-12-09T08:17:17,091 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/.tmp/rep_barrier/a6d5969b3eaf4d8baef6bfd7a5bed22e as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/rep_barrier/a6d5969b3eaf4d8baef6bfd7a5bed22e 2024-12-09T08:17:17,093 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/rep_barrier/a6d5969b3eaf4d8baef6bfd7a5bed22e, entries=6, sequenceid=90, filesize=5.9 K 2024-12-09T08:17:17,094 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/.tmp/table/c5bd80aea8d347a0a5522ecaddf729bb as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/table/c5bd80aea8d347a0a5522ecaddf729bb 2024-12-09T08:17:17,096 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/table/c5bd80aea8d347a0a5522ecaddf729bb, entries=7, sequenceid=90, filesize=5.8 K 2024-12-09T08:17:17,097 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1713ms, sequenceid=90, compaction requested=false 2024-12-09T08:17:17,100 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-09T08:17:17,101 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T08:17:17,101 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T08:17:17,101 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733732235383Running coprocessor pre-close hooks at 1733732235383Disabling compacts and flushes for region at 1733732235383Disabling writes for close at 1733732235383Obtaining lock to block concurrent updates at 1733732235383Preparing flush snapshotting stores in 1588230740 at 1733732235383Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733732235384 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733732235384Flushing 1588230740/info: creating writer at 1733732235385 (+1 ms)Flushing 1588230740/info: appending metadata at 1733732235404 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733732235404Flushing 1588230740/ns: creating writer at 1733732235816 (+412 ms)Flushing 1588230740/ns: appending metadata at 1733732235832 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733732235832Flushing 1588230740/rep_barrier: creating writer at 1733732236239 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733732236251 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733732236251Flushing 1588230740/table: creating writer at 1733732236661 (+410 ms)Flushing 1588230740/table: appending metadata at 1733732236677 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733732236677Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45548fd2: reopening flushed file at 1733732237084 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@746f4ccb: reopening flushed file at 1733732237087 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28a085dd: reopening flushed file at 1733732237090 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44411437: reopening flushed file at 1733732237093 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1713ms, sequenceid=90, compaction requested=false at 1733732237097 (+4 ms)Writing region close event to WAL at 1733732237098 (+1 ms)Running coprocessor post-close hooks at 1733732237101 (+3 ms)Closed at 1733732237101 2024-12-09T08:17:17,101 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T08:17:17,185 INFO [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(976): stopping server 5bda946059d1,43255,1733732044053; all regions closed. 2024-12-09T08:17:17,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741834_1010 (size=25344) 2024-12-09T08:17:17,195 DEBUG [RS:0;5bda946059d1:43255 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/oldWALs 2024-12-09T08:17:17,195 INFO [RS:0;5bda946059d1:43255 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5bda946059d1%2C43255%2C1733732044053.meta:.meta(num 1733732048260) 2024-12-09T08:17:17,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741832_1008 (size=16417043) 2024-12-09T08:17:17,199 DEBUG [RS:0;5bda946059d1:43255 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/oldWALs 2024-12-09T08:17:17,199 INFO [RS:0;5bda946059d1:43255 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5bda946059d1%2C43255%2C1733732044053:(num 1733732047188) 2024-12-09T08:17:17,199 DEBUG [RS:0;5bda946059d1:43255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:17:17,199 INFO [RS:0;5bda946059d1:43255 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:17:17,200 INFO [RS:0;5bda946059d1:43255 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T08:17:17,200 INFO [RS:0;5bda946059d1:43255 {}] hbase.ChoreService(370): Chore service for: regionserver/5bda946059d1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T08:17:17,200 INFO [RS:0;5bda946059d1:43255 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T08:17:17,200 INFO [regionserver/5bda946059d1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T08:17:17,201 INFO [RS:0;5bda946059d1:43255 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43255 2024-12-09T08:17:17,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5bda946059d1,43255,1733732044053 2024-12-09T08:17:17,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T08:17:17,250 INFO [RS:0;5bda946059d1:43255 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T08:17:17,251 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5bda946059d1,43255,1733732044053] 2024-12-09T08:17:17,270 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5bda946059d1,43255,1733732044053 already deleted, retry=false 2024-12-09T08:17:17,271 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5bda946059d1,43255,1733732044053 expired; onlineServers=0 2024-12-09T08:17:17,271 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5bda946059d1,45611,1733732043366' ***** 2024-12-09T08:17:17,271 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T08:17:17,272 INFO [M:0;5bda946059d1:45611 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T08:17:17,272 INFO [M:0;5bda946059d1:45611 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T08:17:17,272 DEBUG [M:0;5bda946059d1:45611 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T08:17:17,273 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T08:17:17,273 DEBUG [M:0;5bda946059d1:45611 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T08:17:17,273 DEBUG [master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.large.0-1733732046909 {}] cleaner.HFileCleaner(306): Exit Thread[master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.large.0-1733732046909,5,FailOnTimeoutGroup] 2024-12-09T08:17:17,273 DEBUG [master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.small.0-1733732046910 {}] cleaner.HFileCleaner(306): Exit Thread[master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.small.0-1733732046910,5,FailOnTimeoutGroup] 2024-12-09T08:17:17,273 INFO [M:0;5bda946059d1:45611 {}] hbase.ChoreService(370): Chore service for: master/5bda946059d1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T08:17:17,273 INFO [M:0;5bda946059d1:45611 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T08:17:17,274 DEBUG [M:0;5bda946059d1:45611 {}] master.HMaster(1795): Stopping service threads 2024-12-09T08:17:17,274 INFO [M:0;5bda946059d1:45611 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T08:17:17,274 INFO [M:0;5bda946059d1:45611 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T08:17:17,274 ERROR [M:0;5bda946059d1:45611 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:38367 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:38367,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-09T08:17:17,276 INFO [M:0;5bda946059d1:45611 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T08:17:17,276 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T08:17:17,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T08:17:17,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:17:17,281 DEBUG [M:0;5bda946059d1:45611 {}] zookeeper.ZKUtil(347): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T08:17:17,281 WARN [M:0;5bda946059d1:45611 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T08:17:17,282 INFO [M:0;5bda946059d1:45611 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/.lastflushedseqids 2024-12-09T08:17:17,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742454_1630 (size=72) 2024-12-09T08:17:17,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:17:17,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43255-0x10009a38e040001, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:17:17,362 INFO [RS:0;5bda946059d1:43255 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T08:17:17,363 INFO [RS:0;5bda946059d1:43255 {}] regionserver.HRegionServer(1031): Exiting; stopping=5bda946059d1,43255,1733732044053; zookeeper connection closed. 2024-12-09T08:17:17,364 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2b69f81f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2b69f81f 2024-12-09T08:17:17,365 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T08:17:17,694 INFO [M:0;5bda946059d1:45611 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T08:17:17,694 INFO [M:0;5bda946059d1:45611 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T08:17:17,694 DEBUG [M:0;5bda946059d1:45611 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T08:17:17,694 INFO [M:0;5bda946059d1:45611 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:17:17,694 DEBUG [M:0;5bda946059d1:45611 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:17:17,694 DEBUG [M:0;5bda946059d1:45611 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T08:17:17,694 DEBUG [M:0;5bda946059d1:45611 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:17:17,694 INFO [M:0;5bda946059d1:45611 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=781.58 KB heapSize=958.20 KB 2024-12-09T08:17:17,706 DEBUG [M:0;5bda946059d1:45611 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5895436e3da1451d87810ba21a10ef64 is 82, key is hbase:meta,,1/info:regioninfo/1733732048343/Put/seqid=0 2024-12-09T08:17:17,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742455_1631 (size=5672) 2024-12-09T08:17:18,110 INFO [M:0;5bda946059d1:45611 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2186 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5895436e3da1451d87810ba21a10ef64 2024-12-09T08:17:18,126 DEBUG [M:0;5bda946059d1:45611 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dcddbbca93284aa3a1cb813944a4cf35 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1733732203404/Put/seqid=0 2024-12-09T08:17:18,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742456_1632 (size=45000) 2024-12-09T08:17:18,530 INFO [M:0;5bda946059d1:45611 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=781.02 KB at sequenceid=2186 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dcddbbca93284aa3a1cb813944a4cf35 2024-12-09T08:17:18,534 INFO [M:0;5bda946059d1:45611 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dcddbbca93284aa3a1cb813944a4cf35 2024-12-09T08:17:18,553 DEBUG [M:0;5bda946059d1:45611 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4a8404c81c5a4dd0885d1b75ca48a9b5 is 69, key is 5bda946059d1,43255,1733732044053/rs:state/1733732046955/Put/seqid=0 2024-12-09T08:17:18,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073742457_1633 (size=5156) 2024-12-09T08:17:18,957 INFO [M:0;5bda946059d1:45611 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2186 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4a8404c81c5a4dd0885d1b75ca48a9b5 2024-12-09T08:17:18,961 DEBUG [M:0;5bda946059d1:45611 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5895436e3da1451d87810ba21a10ef64 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5895436e3da1451d87810ba21a10ef64 2024-12-09T08:17:18,965 INFO [M:0;5bda946059d1:45611 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5895436e3da1451d87810ba21a10ef64, entries=8, sequenceid=2186, filesize=5.5 K 2024-12-09T08:17:18,965 DEBUG [M:0;5bda946059d1:45611 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dcddbbca93284aa3a1cb813944a4cf35 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dcddbbca93284aa3a1cb813944a4cf35 2024-12-09T08:17:18,969 INFO [M:0;5bda946059d1:45611 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dcddbbca93284aa3a1cb813944a4cf35 2024-12-09T08:17:18,969 INFO [M:0;5bda946059d1:45611 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dcddbbca93284aa3a1cb813944a4cf35, entries=168, sequenceid=2186, filesize=43.9 K 2024-12-09T08:17:18,970 DEBUG [M:0;5bda946059d1:45611 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4a8404c81c5a4dd0885d1b75ca48a9b5 as hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4a8404c81c5a4dd0885d1b75ca48a9b5 2024-12-09T08:17:18,973 INFO [M:0;5bda946059d1:45611 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4a8404c81c5a4dd0885d1b75ca48a9b5, entries=1, sequenceid=2186, filesize=5.0 K 2024-12-09T08:17:18,974 INFO [M:0;5bda946059d1:45611 {}] regionserver.HRegion(3140): Finished flush of dataSize ~781.58 KB/800334, heapSize ~957.90 KB/980888, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2186, compaction requested=false 2024-12-09T08:17:18,975 INFO [M:0;5bda946059d1:45611 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:17:18,975 DEBUG [M:0;5bda946059d1:45611 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733732237694Disabling compacts and flushes for region at 1733732237694Disabling writes for close at 1733732237694Obtaining lock to block concurrent updates at 1733732237694Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733732237694Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=800334, getHeapSize=981128, getOffHeapSize=0, getCellsCount=2379 at 1733732237694Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733732237695 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733732237695Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733732237706 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733732237706Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733732238113 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733732238126 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733732238126Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733732238534 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733732238553 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733732238553Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e2e242: reopening flushed file at 1733732238960 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33de2e32: reopening flushed file at 1733732238965 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bbe34aa: reopening flushed file at 1733732238969 (+4 ms)Finished flush of dataSize ~781.58 KB/800334, heapSize ~957.90 KB/980888, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2186, compaction requested=false at 1733732238974 (+5 ms)Writing region close event to WAL at 1733732238974Closed at 1733732238975 (+1 ms) 2024-12-09T08:17:18,977 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b8f08a99-4af6-148a-c263-17b943f08447/MasterData/WALs/5bda946059d1,45611,1733732043366/5bda946059d1%2C45611%2C1733732043366.1733732046248 not finished, retry = 0 2024-12-09T08:17:18,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32793 is added to blk_1073741830_1006 (size=943615) 2024-12-09T08:17:19,078 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T08:17:19,078 INFO [M:0;5bda946059d1:45611 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T08:17:19,079 INFO [M:0;5bda946059d1:45611 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45611 2024-12-09T08:17:19,079 INFO [M:0;5bda946059d1:45611 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T08:17:19,366 INFO [M:0;5bda946059d1:45611 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T08:17:19,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:17:19,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45611-0x10009a38e040000, quorum=127.0.0.1:55863, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:17:19,372 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:17:19,375 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T08:17:19,375 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T08:17:19,376 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T08:17:19,376 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/hadoop.log.dir/,STOPPED} 2024-12-09T08:17:19,380 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T08:17:19,380 WARN [BP-537288734-172.17.0.2-1733732039543 heartbeating to localhost/127.0.0.1:38367 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T08:17:19,380 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T08:17:19,380 WARN [BP-537288734-172.17.0.2-1733732039543 heartbeating to localhost/127.0.0.1:38367 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-537288734-172.17.0.2-1733732039543 (Datanode Uuid a5df0cd5-5ad6-4dc0-92ff-008714c4eaeb) service to localhost/127.0.0.1:38367 2024-12-09T08:17:19,381 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/cluster_81f161a3-853c-b828-244c-d29c3e10103d/data/data1/current/BP-537288734-172.17.0.2-1733732039543 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:17:19,381 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/cluster_81f161a3-853c-b828-244c-d29c3e10103d/data/data2/current/BP-537288734-172.17.0.2-1733732039543 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:17:19,382 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T08:17:19,391 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T08:17:19,392 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T08:17:19,392 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T08:17:19,392 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T08:17:19,392 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e0c4bc29-2a08-3789-b06e-6356b676b0d4/hadoop.log.dir/,STOPPED} 2024-12-09T08:17:19,413 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T08:17:19,583 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down